{"meta":{"start_date":"2026-08-18","end_date":"2026-08-18","start_at":"2026-08-18T00:00:00Z","end_at":"2026-08-19T00:00:00Z","timezone":"UTC","generated_at":"2026-08-25T23:07:21Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":3,"failed_runs":3,"raw_occurrences":52,"matched_occurrences":52,"runs_affected":3},"rows":[{"environment":"prod","failure_pattern_id":"d06f43eb84d7c0e4cc6caaacd92f9c7ebb0caa46f706a4e15263676707ea1c77","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":36,"runs_affected":2,"failed_runs":2,"run_impact_percent":66.66666666666667,"anchor_occurrences":36,"after_last_push_count":165,"trend_counts":[5,4,1,0,0,0,36],"trend_range":"2026-08-12..2026-08-18 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10"],"prior_runs_affected":25,"prior_last_seen_at":"2026-08-14T02:16:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aed260\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-z9m2c6q2wt9f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2381acc6-3e26-4e23-b87b-3b89fac035ca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2381acc6-3e26-4e23-b87b-3b89fac035ca\",\n      \"name\": \"2381acc6-3e26-4e23-b87b-3b89fac035ca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T23:06:57.972257037Z\",\n      \"endTime\": \"2026-08-18T23:25:57.729049787Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:184]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aed260\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-z9m2c6q2wt9f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2381acc6-3e26-4e23-b87b-3b89fac035ca\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2381acc6-3e26-4e23-b87b-3b89fac035ca\",\n      \"name\": \"2381acc6-3e26-4e23-b87b-3b89fac035ca\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T23:06:57.972257037Z\",\n      \"endTime\": \"2026-08-18T23:25:57.729049787Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00054af40\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-vsndz7vcmkc2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8c055328-9528-4d95-bf30-f114c8824240\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8c055328-9528-4d95-bf30-f114c8824240\",\n      \"name\": \"8c055328-9528-4d95-bf30-f114c8824240\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T02:21:25.465764241Z\",\n      \"endTime\": \"2026-08-18T02:40:25.755703486Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00054af40\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-vsndz7vcmkc2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8c055328-9528-4d95-bf30-f114c8824240\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8c055328-9528-4d95-bf30-f114c8824240\",\n      \"name\": \"8c055328-9528-4d95-bf30-f114c8824240\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T02:21:25.465764241Z\",\n      \"endTime\": \"2026-08-18T02:40:25.755703486Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008b94a0\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-t67ltk24vhhl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4b27045c-87f0-45f9-bb70-024ec2e52cbe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4b27045c-87f0-45f9-bb70-024ec2e52cbe\",\n      \"name\": \"4b27045c-87f0-45f9-bb70-024ec2e52cbe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T01:35:16.063344861Z\",\n      \"endTime\": \"2026-08-18T01:54:24.672728151Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008b94a0\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-t67ltk24vhhl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4b27045c-87f0-45f9-bb70-024ec2e52cbe\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4b27045c-87f0-45f9-bb70-024ec2e52cbe\",\n      \"name\": \"4b27045c-87f0-45f9-bb70-024ec2e52cbe\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T01:35:16.063344861Z\",\n      \"endTime\": \"2026-08-18T01:54:24.672728151Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2089849852371406848","occurred_at":"2026-08-18T23:00:06Z","signature_id":"d102826482e4113d48d761f1ada8580b117e1cda924848f0d58d14f224e939d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"041b9aa1ac494f6cd235dc87f8abb9dc909f246b5550b00ac98c7d1424dafc2e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"1f6064d2d14feaffcb29468626955fa32b6e538e40b0347fba636434a9557724","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"2eac66dec7bf14f313360d781e4c751ccbc832dde42410f7465995bb3637cf48","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"388c73a966eb87d13a6b0a890fe517cd0bcf8860d3db0b92ead8ce331caeae3a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"397854bc7a038e9695a0397137da004c93983f8a950df479a759107a3cc350b7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"428bfe04f45e754c0eba10a4521f2088d787ae506f8e0e85f64da65da272c01c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"4d7d389c790fedb98339a6ba13dd9bed3a0dc879558b1fce3f525b58c8ccd9c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"5d2b6914bbf3275c2eb151ae6184f18084177afee3c3532d2f9d7eb408616083","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"657f4980e9dd38d28e0a78d4c26ce05a78c94be99a97c65576350f9dce463536","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"77a4a7473abbefc7aee3c5c3abcc075f102ab2427a8015e17241acc2d3755aea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"7f62cba921b6a83fca905836f915fef862a0b42e45993804cc3552e8d89a761d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"840cc8770a62f72b0a0b11dabcb91665780361842cf566493b2b0ce55407b5e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"8571300fa1257a9fd0b7b6629004f547243f1e44efad3bad8be8675bf140ef5e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"892e81ce4ed37e95b8a7d957b47e55ca0ca53d6afdb829e1bb33b9314db93bac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"89ff037e55a8e22e3f123053c4d148a23715c8ac512b27f63f8f3c413b5cd1cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"8a60db110031773d6b9f423ad140d471edaf25342e94d58f7a36dc55a7757a27","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"8b8a269cb68b60ab7b00650b122aaf1dc5cd55f33c5c53de102618c2ef052975","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"9b745fbedc1e2c99fe9d7b36273eb8671aad0decffb4da47ac9c39ad8454571f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"a194a41cb69234873601b5d7d4824b6c2a1ee1b8eb91cb83915bc5efb321ce44","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"a5813c7f8934fbe4f28c74d94229a0640e82cf197a2bdbea11218aad97721552","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"b1a30a76609d5d3d27983a5b861e33690959716545169541d36294596988972d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"bf7dcf565f0f2a2957f82691b221d58b3cefd2a66fb5c69c090b3c18ea8ec7ee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"c68652b7b1657b9d5609c0ba46f3f3668286f595f5934a45c115b641d7b93e29","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"c7bce42e4c8d8d606848ff8f7e90c1c27337f7519189a7c383270bbbbd3d8e59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"cae8e36ec0327d10cd7f99fc9c5d43fe776765c292f37c549269d4db4c8af29c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"cb80aa4fa3b3d3f61f84afb895c0c1837aefb9c829282e41ab20ffd0d224d3a4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"d78649e1fd9154ebf96d2faba3860c1d2ac5c534a98bb7b5bc4affa1351089c8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"dfeb539901ecaa07c3030ff6728b368dbca357f9c4c07dc99536c7bd920ef6c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"e51f6cee86aae5c95401f4477176f4a9ad3db5adfdacc22ddcafd067a1e2d245","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"ead94180f2a4c2f340b11f7aeccacdb870b8bf760d47a749bca876e4d31a74b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"ec3208875d30fe5bc87b02fe17e995f2ea04034d671a588bce72dc22039224c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"ed115fada96b83159176296eb2e9fd5e3a32ae4a8db7626547911f0fb618469b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"ee4741f818d06d65a5a54f104f1f4992219c9c418a462448370d6729e0122666","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"eecf676b63e663b955a0f0873b4e5ef24056571303e07100262c0b39f3f2ffed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"fcef4edcff9b374e839d9048c2154c98d5fd846261391a89edd7bab0bd4f43a4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":8,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":8,"after_last_push_count":85,"trend_counts":[0,44,0,0,0,0,8],"trend_range":"2026-08-12..2026-08-18 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-27","2026-08-03","2026-08-10"],"prior_runs_affected":7,"prior_last_seen_at":"2026-08-13T12:05:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260630preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-06-30 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a94b00\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-rl7558nqx6x5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6c83ecdf-321d-4c22-8c0e-0e67dfc51b95\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6c83ecdf-321d-4c22-8c0e-0e67dfc51b95\",\n      \"name\": \"6c83ecdf-321d-4c22-8c0e-0e67dfc51b95\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T02:22:38.138134816Z\",\n      \"endTime\": \"2026-08-18T02:41:45.799493023Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a94b00\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-rl7558nqx6x5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6c83ecdf-321d-4c22-8c0e-0e67dfc51b95\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6c83ecdf-321d-4c22-8c0e-0e67dfc51b95\",\n      \"name\": \"6c83ecdf-321d-4c22-8c0e-0e67dfc51b95\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T02:22:38.138134816Z\",\n      \"endTime\": \"2026-08-18T02:41:45.799493023Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-20-hgtq5vkjmtng/cluster-candidate-4-20-7ntjrh should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005ac5a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-7ntjrh: failed waiting for cluster=\"cluster-candidate-4-20-7ntjrh\" in resourcegroup=\"rg-candidate-4-20-hgtq5vkjmtng\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7498cc37-bb15-4954-891a-544689c57fd9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7498cc37-bb15-4954-891a-544689c57fd9\",\n      \"name\": \"7498cc37-bb15-4954-891a-544689c57fd9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T02:20:22.991368241Z\",\n      \"endTime\": \"2026-08-18T02:39:25.761048408Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-candidate-4-20-hgtq5vkjmtng/cluster-candidate-4-20-7ntjrh should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005ac5a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-7ntjrh: failed waiting for cluster=\"cluster-candidate-4-20-7ntjrh\" in resourcegroup=\"rg-candidate-4-20-hgtq5vkjmtng\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7498cc37-bb15-4954-891a-544689c57fd9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7498cc37-bb15-4954-891a-544689c57fd9\",\n      \"name\": \"7498cc37-bb15-4954-891a-544689c57fd9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T02:20:22.991368241Z\",\n      \"endTime\": \"2026-08-18T02:39:25.761048408Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008c74a0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-klf4n4fvlz26\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d5fa142d-cb72-43e2-a5ac-bdaf2c8ab514\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d5fa142d-cb72-43e2-a5ac-bdaf2c8ab514\",\n      \"name\": \"d5fa142d-cb72-43e2-a5ac-bdaf2c8ab514\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T01:35:55.986111403Z\",\n      \"endTime\": \"2026-08-18T01:55:04.389818569Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008c74a0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-klf4n4fvlz26\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d5fa142d-cb72-43e2-a5ac-bdaf2c8ab514\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d5fa142d-cb72-43e2-a5ac-bdaf2c8ab514\",\n      \"name\": \"d5fa142d-cb72-43e2-a5ac-bdaf2c8ab514\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T01:35:55.986111403Z\",\n      \"endTime\": \"2026-08-18T01:55:04.389818569Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"1c6824d7b323fce6d653b2a7ce27d98832d2ddc2ff92872a85e019181c2e8924","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"1f0a19672803d999645185907742387cf6347669800e68ab21450ca003260aa1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"2395cef2fccc79d8e7bbb73689c161c5d875f0f9169e2b9d6e6c1d449be4106d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"3f8a8c981c1a8e22c7c19bf9cf1e5752615c6cb47c4a059c117462d449fc3da7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"461e1a912b7bcea4d8eb3836e752b3387a785782112d0fb1a0858addb3ffdc8d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"560d6bfcb47daf5b5489be029b573902284f2de8a40e5ea2fbcb254c49dfed26","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"6ae87330291c79960e8d92078b1cd730312ecfdb9817b515d823377e9245692b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"f58c68d35bb7143d6ec3c290d1364a705a65880a610832ade77ff181382c7a4a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-22-cxtdzk, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-cxtdzk in resource group rg-candidate-4-22-9rh5s6cz8tvc, error: failed waiting for cluster=\"cluster-candidate-4-22-cxtdzk\" in resourcegroup=\"rg-candidate-4-22-9rh5s6cz8tvc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-cxtdzk in resource group rg-candidate-4-22-9rh5s6cz8tvc, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":5,"runs_affected":2,"failed_runs":2,"run_impact_percent":66.66666666666667,"anchor_occurrences":5,"after_last_push_count":34,"trend_counts":[2,3,2,2,2,2,5],"trend_range":"2026-08-12..2026-08-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-20","2026-07-27","2026-08-03","2026-08-10"],"prior_runs_affected":23,"prior_last_seen_at":"2026-08-16T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001c36390\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-n2w89h2n5hbn, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-n2w89h2n5hbn\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-n2w89h2n5hbn, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001c36390\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-n2w89h2n5hbn, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-n2w89h2n5hbn\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-n2w89h2n5hbn, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-1-mds7bfdnr4mb/cluster-nightly-5-1-jkzd98 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012549c0\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-jkzd98, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-jkzd98 in resource group rg-nightly-5-1-mds7bfdnr4mb, error: failed waiting for cluster=\"cluster-nightly-5-1-jkzd98\" in resourcegroup=\"rg-nightly-5-1-mds7bfdnr4mb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-jkzd98 in resource group rg-nightly-5-1-mds7bfdnr4mb, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-1-mds7bfdnr4mb/cluster-nightly-5-1-jkzd98 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012549c0\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-jkzd98, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-jkzd98 in resource group rg-nightly-5-1-mds7bfdnr4mb, error: failed waiting for cluster=\"cluster-nightly-5-1-jkzd98\" in resourcegroup=\"rg-nightly-5-1-mds7bfdnr4mb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-jkzd98 in resource group rg-nightly-5-1-mds7bfdnr4mb, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:151]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013d8240\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-qfcrpn6hrh9n\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-qfcrpn6hrh9n, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:151]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013d8240\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-qfcrpn6hrh9n\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-qfcrpn6hrh9n, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2089849852371406848","occurred_at":"2026-08-18T23:00:06Z","signature_id":"cc47eb855a607ee9689d0e169d40bdfd378887a3dcba5ef2ab82f3db469c5974","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2089849852371406848","occurred_at":"2026-08-18T23:00:06Z","signature_id":"d7305db7bd27d6f90fe8bd6a0dba4e4fb9466ea5946ec562b845a26d40ddc37c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"a667de98947b6324d1e144a014a7ca47dfafe783ef29a1944cdd8d144610abf1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"ac73ed7a746449db979e0e2261907b0458af3d9b801b03787837f6bcdfb9de11","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"fd3ca4f0abe6b38cf9a8288cea30fe69cd19f4c2b6aeb4087204acf6233efaa7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":6,"trend_counts":[0,2,0,0,0,0,1],"trend_range":"2026-08-12..2026-08-18 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-20","2026-07-27","2026-08-10"],"prior_runs_affected":10,"prior_last_seen_at":"2026-08-13T16:29:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c14df85165f7ab51105a1e05c6306a467e1e1edcac57869d6c32690164316413","failure_pattern":"VerifyVMOSDiskCustomerEncryption(nodePool=\u003cnodepool\u003e) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-z8ktk-l4d4d_OSDisk for VM des-encrypt-des-np-z8ktk-l4d4d has encryption type EncryptionAtRestWithPlatformKey, expected","search_query":"VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-z8ktk-l4d4d_OSDisk for VM des-encrypt-des-np-z8ktk-l4d4d has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-12..2026-08-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:163]: cluster verification failed\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013905a0\u003e: \n    VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-z8ktk-l4d4d_OSDisk for VM des-encrypt-des-np-z8ktk-l4d4d has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    disk des-encrypt-des-np-z8ktk-t2cjq_OSDisk for VM des-encrypt-des-np-z8ktk-t2cjq has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:163]: cluster verification failed\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013905a0\u003e: \n    VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-z8ktk-l4d4d_OSDisk for VM des-encrypt-des-np-z8ktk-l4d4d has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    disk des-encrypt-des-np-z8ktk-t2cjq_OSDisk for VM des-encrypt-des-np-z8ktk-t2cjq has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089532925845442560","occurred_at":"2026-08-18T02:00:45Z","signature_id":"79876917e6824dcbd8ac712a77e0a6fb9ff34a76687e955fa8cb42c1b782a145","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"26ef93b7bfdd8d13fd9f4e8081e341d2e56fd5625c71012d2c9838c48a257ac2","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":22,"trend_counts":[4,4,1,0,0,0,1],"trend_range":"2026-08-12..2026-08-18 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-20","2026-07-27","2026-08-03","2026-08-10"],"prior_runs_affected":31,"prior_last_seen_at":"2026-08-14T02:16:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:220]: Timed out after 2520.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:216 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2089521975687909376","occurred_at":"2026-08-18T01:17:15Z","signature_id":"9013e675989bd3312d95cfd426e5318859765876455d6850dc98ee4d503feff7","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}