{"meta":{"start_date":"2026-05-13","end_date":"2026-05-13","start_at":"2026-05-13T00:00:00Z","end_at":"2026-05-14T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T14:40:04Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":2,"failed_runs":2,"raw_occurrences":40,"matched_occurrences":40,"runs_affected":2},"rows":[{"environment":"prod","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":26,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":26,"after_last_push_count":28,"trend_counts":[0,0,0,0,0,0,26],"trend_range":"2026-05-07..2026-05-13 UTC","prior_windows_present":2,"prior_window_starts":["2026-04-13","2026-04-20"],"prior_runs_affected":4,"prior_last_seen_at":"2026-04-24T04:34:07Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:249]: Unexpected error:\n    \u003c*errors.joinError | 0xc000914120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-l8jbbf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-7mdv5l\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:249]: Unexpected error:\n    \u003c*errors.joinError | 0xc000914120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-l8jbbf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-candidate-4-21-7mdv5l\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:249]: Unexpected error:\n    \u003c*errors.joinError | 0xc002316300\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-9t7gt7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:249]: Unexpected error:\n    \u003c*errors.joinError | 0xc002316300\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-9t7gt7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:249]: Unexpected error:\n    \u003c*errors.joinError | 0xc0016deae0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-8d74qd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:249]: Unexpected error:\n    \u003c*errors.joinError | 0xc0016deae0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-8d74qd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"0b584415d65ca756dc1f852cacb82c8b1cb0771ff14ce6aca794848b692dfac0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"0e092dfc089508238761c8154527c066ca093c794f78ec3658e5fc62fda41fca","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"1a523784fc17094432cbebea9ef3c3990e7e65e3160d39cce27b5493c26bc7b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"1b01a2358b1324e52d86774e3d09fe12e8ce57538b93849d5b14c5dad8c3a99f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"3eb46e2b8d29c75fdd9d5600533633868ab21b6781653b095cd0d00e18845629","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"4343bda930af1d4d130f3c5d1c4af7157200b428137822b6a64dfa2a7da41f72","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"4774e96c1f08da60fecbee19ef267c6e5a6595db2258a389c16c4d71595be2f2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"4d86dfb4ce7427564a25d2034c682684abb26a6dd157e0fc1c1f78ec678b1d49","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"5827049d7e6492868366d0ae5760056ccfd29d36da41ccdf8f261bfc2278eef0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"721ebfd6b67998009a5c051c5d25b511a9a7fa5726da760dfe593b5f203799c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"82422424c94ef75bd336c1582883293b59b03a18b76921e85ee996e9a093936a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"897b6d42d29508a7e0d75d0b6f591e563ec2f30e6d9aadcbb3b5f18e4bec14df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"9d0071c8e13276d0a4d84c28e0e1b88a94149f5fe6bb6ee9d7e3f52e00851d78","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"a6747560da9449a2f2e81ea68acab503d2a38cf0a9cf363fd67b9e2648a200ed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"a9f1f812400bbd6039386e30f1d8dc0fc8758e0c34b0d4a8cecb2976037597c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"ad6c83aca6f0f44b891dee02bf7acd3bbd411f983fe4f3d3ae9306faf8730375","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"b9460cc801d5b768238c66284013cff0e867ddc85d18f6e23740f062d5727ea2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"c4290dab4432159dc398b66d53c867be46f0b74fa3ba45433cfd912ec410f87a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"c7fb262dbbb788f27970a30428d1e85a3513a9aa4a15836e86ce1003b6a671e7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"e2b446673f23a21f630814838e42b7f2fe822538c3cf8a7407c0e8228b205de0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"e54de82749ecdadc090c41d9e6440642e72bcdd5f0b12ee5381bb7c56e3df56f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"f6d978c9b54069debe7f2eec58c07f327faa76854adf97b4eddf940ee904a7a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"f961b814cd69f893234b662be65bfaab8ee0272eadc6af27dbed6c78a917d8f8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"faf66d2d6fc1c695df517a3cc97b3a83a23af9a04ebb049f53116ebf5c273ed8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"fb3c65762199e5caee5359245295ccefb0593299a8ce0b0dfca21d26d9112172","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"ff8b606eafc646464a00e9ea418a489caef8707475870d411997392c1ea89c4f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-52hx56, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-52hx56\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-52hx56, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":10,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":10,"after_last_push_count":320,"trend_counts":[12,37,0,0,0,1,10],"trend_range":"2026-05-07..2026-05-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-04-13","2026-04-20","2026-04-27","2026-05-04"],"prior_runs_affected":27,"prior_last_seen_at":"2026-05-08T17:54:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:130]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001372630\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-499vgf, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-499vgf in resource group rg-np-version-upgrade-499vgf-92gnjm, error: failed waiting for cluster=\"np-version-upgrade-cluster-499vgf\" in resourcegroup=\"rg-np-version-upgrade-499vgf-92gnjm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-499vgf in resource group rg-np-version-upgrade-499vgf-92gnjm, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:130]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001372630\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-499vgf, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-499vgf in resource group rg-np-version-upgrade-499vgf-92gnjm, error: failed waiting for cluster=\"np-version-upgrade-cluster-499vgf\" in resourcegroup=\"rg-np-version-upgrade-499vgf-92gnjm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-499vgf in resource group rg-np-version-upgrade-499vgf-92gnjm, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00156ca50\u003e: \n    failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-xdbkw8, error: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-xdbkw8\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-xdbkw8, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00156ca50\u003e: \n    failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-xdbkw8, error: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-xdbkw8\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-xdbkw8, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:109]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011c3230\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-k2l6vh, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-candidate-4-20-k2l6vh in resource group rg-candidate-4-20-x2t2g8, error: failed waiting for cluster=\"cluster-candidate-4-20-k2l6vh\" in resourcegroup=\"rg-candidate-4-20-x2t2g8\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-candidate-4-20-k2l6vh in resource group rg-candidate-4-20-x2t2g8, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:109]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011c3230\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-k2l6vh, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-candidate-4-20-k2l6vh in resource group rg-candidate-4-20-x2t2g8, error: failed waiting for cluster=\"cluster-candidate-4-20-k2l6vh\" in resourcegroup=\"rg-candidate-4-20-x2t2g8\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-candidate-4-20-k2l6vh in resource group rg-candidate-4-20-x2t2g8, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"030923b4469f49c276ffa3b9d92dd0cf199e43d610be3cffa33cc36d47d7dcfc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"259c0c4878a3294064ff783427a9495335289af4a76e2c2969548d6d7488f93b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"3dfcc3db6e67421d715c86030d8c6f40321e1e0e9b5bcccb610633797a87a7fe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"44d55884625a502599ab8997cb52ee4ea61cf80a6abe3811fc1118481f9c1609","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"5357482ff042392fbea7d971b3d746379a08250e4eda2d00cc87e2afdf6179d7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"64d6c6f613a99bfd4ac4c8fc12ba47f3a00c3ee4f88934fd14baba0eaf3935e3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"690ef8ee25232a11d83d13e985fc55e2beae046272b8bf29cf0835a919334a1d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"6e54c815df2350b1a97a61536eea7c7b1ab0538e2f3ee5777dae47741175f3f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"b33e3f53b73db25f34c0bf8198361fdc365adc906107819227046e2433841085","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"e17c66a32ce4618afc697d2fbd63321cbed74d3295fef46f766daf20d76c35ce","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"0fa92f751490b05776d8c9d0ffad7e078841d629f2bbb5cf7b86f379f22f1b39","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message timed out after 29s waiting for resolution of desired version from '4.21.15' cluster version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":100,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-05-07..2026-05-13 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:160]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0014084a0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-5wk752\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-qwr4c4\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/423d54f5-9311-40ba-a0fa-52e5d5e4dcd8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/423d54f5-9311-40ba-a0fa-52e5d5e4dcd8\",\n      \"name\": \"423d54f5-9311-40ba-a0fa-52e5d5e4dcd8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-05-13T13:46:24.415720989Z\",\n      \"endTime\": \"2026-05-13T13:48:12.932582258Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"timed out after 29s waiting for resolution of desired version from '4.21.15' cluster version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:160]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0014084a0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-5wk752\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-qwr4c4\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/423d54f5-9311-40ba-a0fa-52e5d5e4dcd8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/423d54f5-9311-40ba-a0fa-52e5d5e4dcd8\",\n      \"name\": \"423d54f5-9311-40ba-a0fa-52e5d5e4dcd8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-05-13T13:46:24.415720989Z\",\n      \"endTime\": \"2026-05-13T13:48:12.932582258Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"timed out after 29s waiting for resolution of desired version from '4.21.15' cluster version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:160]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000598220\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-fwht97\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-wn6jfz\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4f5e233c-1544-4fcf-acfe-ac988c79aefa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4f5e233c-1544-4fcf-acfe-ac988c79aefa\",\n      \"name\": \"4f5e233c-1544-4fcf-acfe-ac988c79aefa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-05-13T02:53:03.508575142Z\",\n      \"endTime\": \"2026-05-13T02:54:22.582271334Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"timed out after 29s waiting for resolution of desired version from '4.21.15' cluster version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:160]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000598220\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-fwht97\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-wn6jfz\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4f5e233c-1544-4fcf-acfe-ac988c79aefa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4f5e233c-1544-4fcf-acfe-ac988c79aefa\",\n      \"name\": \"4f5e233c-1544-4fcf-acfe-ac988c79aefa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-05-13T02:53:03.508575142Z\",\n      \"endTime\": \"2026-05-13T02:54:22.582271334Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"timed out after 29s waiting for resolution of desired version from '4.21.15' cluster version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"32c5eaec9aee930a7246240a8b34ba30a49776a7d954a5f9bec99120da53877c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2054382242708328448","occurred_at":"2026-05-13T02:00:06Z","signature_id":"32450619c037bc494079db40c33ca0044f5df5558d9030d23cf045bacda24022","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ae6a7f7d2e3c3444ed20150a67378f751586e201de3a927f677f72a871b2bf44","failure_pattern":"DeleteResourceGroup timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-zg8gxb, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-05-07..2026-05-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-04-20"],"prior_runs_affected":2,"prior_last_seen_at":"2026-04-24T18:17:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:143]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000490e40\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-zg8gxb, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:143]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000490e40\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-zg8gxb, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2054545360491122688","occurred_at":"2026-05-13T12:00:01Z","signature_id":"b8bd9012025eb327d93ee57bab10dac7f7e9844f919d1392e7bf6f9a26973f0d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-05-07..2026-05-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-04-20"],"prior_runs_affected":3,"prior_last_seen_at":"2026-04-24T18:17:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0005ab2e0\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-ct2x6.apps.aro.y8e3k0c7u4p5b7t.h4i0.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:173]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0005ab2e0\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-ct2x6.apps.aro.y8e3k0c7u4p5b7t.h4i0.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2054382242708328448","occurred_at":"2026-05-13T02:00:06Z","signature_id":"579d418056e7a8f42a03faa9816418f84d18ed55d09b4b4a5e792a175c4e6291","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}