{"meta":{"start_date":"2026-05-11","end_date":"2026-05-17","start_at":"2026-05-11T00:00:00Z","end_at":"2026-05-18T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T11:13:53Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":31,"failed_runs":15,"raw_occurrences":79,"matched_occurrences":79,"runs_affected":15},"rows":[{"environment":"int","failure_pattern_id":"0ece011306dab74cd22dd19f1f3c89bbdb930a2321b61be59e4a6a90aae6e3eb","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-8ldb78, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-8ldb78\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-8ldb78, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":39,"runs_affected":8,"failed_runs":8,"run_impact_percent":25.806451612903224,"anchor_occurrences":39,"after_last_push_count":39,"trend_counts":[0,1,15,2,21,0,0,0],"trend_range":"2026-05-10..2026-05-17 UTC","prior_windows_present":4,"prior_window_starts":["2026-04-13","2026-04-20","2026-04-27","2026-05-04"],"prior_runs_affected":23,"prior_last_seen_at":"2026-05-08T18:38:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes vm type NC4asT4v3 in a single cluster","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013269f0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-8ldb78, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-8ldb78\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-8ldb78, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013269f0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-8ldb78, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-8ldb78\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-8ldb78, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000bcd1d0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-6r4k9b, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-6r4k9b in resource group rg-zstream-upgrade-4-21-s6x9zc, error: failed waiting for cluster=\"cluster-zstream-4-21-6r4k9b\" in resourcegroup=\"rg-zstream-upgrade-4-21-s6x9zc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-6r4k9b in resource group rg-zstream-upgrade-4-21-s6x9zc, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000bcd1d0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-6r4k9b, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-6r4k9b in resource group rg-zstream-upgrade-4-21-s6x9zc, error: failed waiting for cluster=\"cluster-zstream-4-21-6r4k9b\" in resourcegroup=\"rg-zstream-upgrade-4-21-s6x9zc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-6r4k9b in resource group rg-zstream-upgrade-4-21-s6x9zc, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:130]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016a43c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-t9cqjk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-t9cqjk in resource group rg-np-version-upgrade-t9cqjk-n82ftb, error: failed waiting for cluster=\"np-version-upgrade-cluster-t9cqjk\" in resourcegroup=\"rg-np-version-upgrade-t9cqjk-n82ftb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-t9cqjk in resource group rg-np-version-upgrade-t9cqjk-n82ftb, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:130]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016a43c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-t9cqjk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-t9cqjk in resource group rg-np-version-upgrade-t9cqjk-n82ftb, error: failed waiting for cluster=\"np-version-upgrade-cluster-t9cqjk\" in resourcegroup=\"rg-np-version-upgrade-t9cqjk-n82ftb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-t9cqjk in resource group rg-np-version-upgrade-t9cqjk-n82ftb, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2055037502590291968","occurred_at":"2026-05-14T21:28:15Z","signature_id":"9eaa8d74f13b4f3ace070f266d1cb1bdfa71deee5e82f3b7c4517e97b2f5c458","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"06a5f88867c35d1df0e300b3790a87bea5f8884b2f0d8e7f39983e486a23a582","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"1d0b062351c390420bd6a7fbd3214056756d79ebe52a736086015527f9c0e375","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"38d0f7d434d3d5dfd37a81e737b938af182d3115b9d4c71c3ad5b9163dfe9d0c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"5633c5035b8548ae6d6d461391a9381107d902e092f93688b08e5ed3e4ed6ab8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"574b84be02a3288a74904fdcff082b8ede1a49310701ad5d3ba313dd5cde8396","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"6582d0e2cf2ba35fbffae0a9e9d10c6945ca98894f93460c3a4e1c1330fc903a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"7825c3614613367a3b7f656dafeb2c17321ddaa8a9dd1bb0927fb72c1732f006","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"815116a8eee64aec5fcff8d4df53b68e237d0ad64a22ccce6add780a72e683d3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"83e5f252c405dca0c65f4bb154b7a79b894c7d4724f7bda1f34b6a1a699cf8e6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"8bcef61ead2dcd67687e827a45fcb0ca56ec19ac9c444dd8aac48f9455872833","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"a1ef1e1a2d7c4f7b6bc4a6cd4e8dbf3dd47e0d82c08e0245649f0bda618b6d53","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"a3d746c244e71b86329ad75ff8570b5e6fd6d24c8048f4f6a3e500287f308935","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"b0fb94ea60dee3450350024732cd741c7c6609e591db7ba19fe547a00a5cb16d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"bab1425ae35fc894918ca5de241735c2f0cf8740bf9c50345c9782fd862cbf0e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"bd5213e85cb223e04092498ba721a3512a918f669a143a292a469afae7de48b7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"c26bd7ee8e74a65f7251a41b8c73a636db54b96f1be61ecbe2808a5fe6d1550e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"caeab39c48e3f17ed84da9b7657a6023fe8406757a609bd0efe9004c32a7ca6a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"d35985e5ba5ec1c816fcedda2bb4a9943d421651986b78954b068bde1d19d9b9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"ebbcf63133d3d9fb7481757e7db2280ce10a3d20d5a84df72fb86aab84bae2cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"f097f69c3a95f439357b736ab90312941eebfd2235b81a5307dbfe05ca6a774d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054651356416839680","occurred_at":"2026-05-13T19:53:51Z","signature_id":"25f9e03fd593ef52b68d896951574d1380d7d703f742e1fecff9f718bbf325b4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054581570538311680","occurred_at":"2026-05-13T15:16:27Z","signature_id":"ea99ada7b9391bbef4d38a79ed01c3daa3eefa278b5316917f7e9a21bc1b0f30","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054338636807147520","occurred_at":"2026-05-12T23:10:19Z","signature_id":"04240422ebac310affd0e4016feaf037d3ea95f22995cbc7cee2c653b2e19f6f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054338636807147520","occurred_at":"2026-05-12T23:10:19Z","signature_id":"1675ff65a6e4ad382caee62948c6454e3157c8d67f9d84fe6343677248329a59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054338636807147520","occurred_at":"2026-05-12T23:10:19Z","signature_id":"3b435a773c4f0dafd28edf258e365f98c2f6e267794643444d947a9b9cfac896","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054338636807147520","occurred_at":"2026-05-12T23:10:19Z","signature_id":"43888281f9506dc70a7bfdc50df67033dbc80adca015132fb8bf0e04dbb55cfa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054338636807147520","occurred_at":"2026-05-12T23:10:19Z","signature_id":"d77610f749f188b7be694563e7c163afb59c385eb6202cc1aa2f06ad04c738d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054157437681799168","occurred_at":"2026-05-12T11:11:12Z","signature_id":"5f9ce836f2fc20b28580e93bf080cabb562d0a2bddc43c5f98473f9e6912c2f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054157437681799168","occurred_at":"2026-05-12T11:11:12Z","signature_id":"75ea8c74219865d5b1983fc3c5de5cd872e979b4a14f35e5dbc4ab3f7317116a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054157437681799168","occurred_at":"2026-05-12T11:11:12Z","signature_id":"8845da17a9c0bb91fef558e645df14df4966399b167d44f55f37943a6a121d5b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054157437681799168","occurred_at":"2026-05-12T11:11:12Z","signature_id":"9d0825c48d2da9534a3cd4b39d9a8e742887cc2a0d11eabcd819b9a19e65f86b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054157437681799168","occurred_at":"2026-05-12T11:11:12Z","signature_id":"d504c5e2f784687ab0a1f10f29d27007dd0cef056186ae0c2de4f978ead1b41c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054038477145640960","occurred_at":"2026-05-12T03:18:29Z","signature_id":"2f3d1834f7e146156fe437f31780e3d42aa9bb438aafcc33356bbc791bc75e9f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054038477145640960","occurred_at":"2026-05-12T03:18:29Z","signature_id":"454a20ba2164a120b91247aad7a0af87ef7fe3be8cc55b2ce48c92655401ae55","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054038477145640960","occurred_at":"2026-05-12T03:18:29Z","signature_id":"876401e39e5e162efeef5510292a8d38dca6136beddc88a2de13950f40e6b31c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054038477145640960","occurred_at":"2026-05-12T03:18:29Z","signature_id":"b7597321ea3fb393f81a888f51d0c3383381c3f6c4706846c208bf4908260a2a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054038477145640960","occurred_at":"2026-05-12T03:18:29Z","signature_id":"f83a2d39cb133081182dd98514f4c763d5af7cc9da9cd16db6c9b9bd3e2c5ba3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2053786945942720512","occurred_at":"2026-05-11T10:39:00Z","signature_id":"29242c86b8fc02265fb5c2f2295358c1cb6c78b948341f7628a02cd1909c1fb1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"52d9258f03a60774b49eaa70a514574e25b40d0eeb6ddfa13f252e24164f6be2","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-9mdxlx\" in resourcegroup=\"admin-credential-lifecycle-test-5szj8p\" to finish deleting: context canceled","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":20,"runs_affected":8,"failed_runs":8,"run_impact_percent":25.806451612903224,"anchor_occurrences":20,"after_last_push_count":20,"trend_counts":[0,1,10,7,2,0,0,0],"trend_range":"2026-05-10..2026-05-17 UTC","prior_windows_present":0,"prior_runs_affected":8,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes vm type NC4asT4v3 in a single cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00171a558\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-cj2npq\" in resourcegroup=\"rg-np-version-upgrade-cj2npq-b8gwnd\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00171a558\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-cj2npq\" in resourcegroup=\"rg-np-version-upgrade-cj2npq-b8gwnd\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001c08360\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-jfrqxb\" in resourcegroup=\"rg-zstream-upgrade-4-20-wwwdbx\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001c08360\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-jfrqxb\" in resourcegroup=\"rg-zstream-upgrade-4-20-wwwdbx\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:249] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001312198\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-9mdxlx\" in resourcegroup=\"admin-credential-lifecycle-test-5szj8p\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:249] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001312198\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-9mdxlx\" in resourcegroup=\"admin-credential-lifecycle-test-5szj8p\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2055037502590291968","occurred_at":"2026-05-14T21:28:15Z","signature_id":"5202052add0b795a81215c941dd88792de0c42f4ea36362d9fb0693e22b0e015","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054979965346123776","occurred_at":"2026-05-14T17:39:38Z","signature_id":"e4cae0392febcb2e2061ff00bab566e51107ecbeed46bf72ab7c22551560b7ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054702578100342784","occurred_at":"2026-05-13T23:17:23Z","signature_id":"804b064328e4f5d5bdcf84d7ccc88e58729aed7f0242a82b80dd93b8613122b3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054702578100342784","occurred_at":"2026-05-13T23:17:23Z","signature_id":"9969e8478625853f0f157c5e12fa5e1c4f4e78c5ceb1e070e74f4f955e88608d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054702578100342784","occurred_at":"2026-05-13T23:17:23Z","signature_id":"fb8bd64b06f951f3b5189a67d035c30530f75856f91844fc6d0e73f37ff26919","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054651356416839680","occurred_at":"2026-05-13T19:53:51Z","signature_id":"0785385e95498ce8f49d73b7e8dc08dc89333fc20eb22e79310882018fac4e1d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054651356416839680","occurred_at":"2026-05-13T19:53:51Z","signature_id":"828841ea953ef5b091303c0afe4a125d72cae2469de878f87a053f17ddb4e6c3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054651356416839680","occurred_at":"2026-05-13T19:53:51Z","signature_id":"932bd293205dc518babef8f46ef4dfbaa0d0ebd002cd205794b136ddde739b79","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054651356416839680","occurred_at":"2026-05-13T19:53:51Z","signature_id":"b5018427fb00da2955ca8f2222b5c18535ab44a253c96a8c7b969ce27a2418af","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054338636807147520","occurred_at":"2026-05-12T23:10:19Z","signature_id":"338a7c9bccc917956cd8421e865384a90a5dd5e99992a1d4285aadd2a2c08a7b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054338636807147520","occurred_at":"2026-05-12T23:10:19Z","signature_id":"d78b9c234498c34762ffe7fbd583d3b30a0a9f2a57c624e7e573eb8032beb5ee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054157437681799168","occurred_at":"2026-05-12T11:11:12Z","signature_id":"71003d5e4ce3b15d628bd3639d7193c8e81bbeecaae89fbe4c305e87bf5175a9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054157437681799168","occurred_at":"2026-05-12T11:11:12Z","signature_id":"75f42982febd59ba9356e6114522bf4d200cdac123973e55bb338d32a70f7c54","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054157437681799168","occurred_at":"2026-05-12T11:11:12Z","signature_id":"a346f2aa1bec3ed70641a543d2a1682fe737711ac9c1c5c7d08f7504a05a9ba3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054157437681799168","occurred_at":"2026-05-12T11:11:12Z","signature_id":"d3083389fff2729493c7c80e1973b2f9bddcd6c006f1c0fd5fbc50df0650b053","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054157437681799168","occurred_at":"2026-05-12T11:11:12Z","signature_id":"f43b7432bda7bfe7a7978c7ea3c95d3e979688a85fceebf9261594435ec0398a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054038477145640960","occurred_at":"2026-05-12T03:18:29Z","signature_id":"13b441d736ded12d806c3502dcfd7df61cd6ede5f891821bd194fa3c7396a953","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054038477145640960","occurred_at":"2026-05-12T03:18:29Z","signature_id":"e41a91c38d00f9e12af3a44b877770e70bdcd48b60c3b60d3df96326eb4ffb7c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054038477145640960","occurred_at":"2026-05-12T03:18:29Z","signature_id":"f0a3167e3b1fbe228dd61a7df1862469c988e126d0995039a4026ca5c13e7b7c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2053786945942720512","occurred_at":"2026-05-11T10:39:00Z","signature_id":"350978b208e70da8347a4a79746580df14fbf49019e1262e871a278c79acf4cd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"9450c3510fa1b3d6bd923dc877507dc8addabc8ce5b76024f64036b5eb8d4b2e","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message timed out after 29s waiting for resolution of desired version from '4.21.15' cluster version; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":19.35483870967742,"anchor_occurrences":6,"after_last_push_count":6,"trend_counts":[0,0,0,6,0,0,0,0],"trend_range":"2026-05-10..2026-05-17 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":6}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:160]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0008449c0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-x9glvp\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-k7tb76\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/82580c93-80ca-4782-a780-8beefa3ca171\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/82580c93-80ca-4782-a780-8beefa3ca171\",\n      \"name\": \"82580c93-80ca-4782-a780-8beefa3ca171\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-05-14T00:41:19.022062955Z\",\n      \"endTime\": \"2026-05-14T00:42:47.115197921Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"timed out after 29s waiting for resolution of desired version from '4.21.15' cluster version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:160]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0008449c0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-x9glvp\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-k7tb76\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/82580c93-80ca-4782-a780-8beefa3ca171\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/82580c93-80ca-4782-a780-8beefa3ca171\",\n      \"name\": \"82580c93-80ca-4782-a780-8beefa3ca171\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-05-14T00:41:19.022062955Z\",\n      \"endTime\": \"2026-05-14T00:42:47.115197921Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"timed out after 29s waiting for resolution of desired version from '4.21.15' cluster version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:160]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000c94560\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-gjf5t9\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-fg6lv9\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f247cdd-5812-4761-b99d-f988d2cb1453\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f247cdd-5812-4761-b99d-f988d2cb1453\",\n      \"name\": \"5f247cdd-5812-4761-b99d-f988d2cb1453\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-05-13T21:07:58.388882897Z\",\n      \"endTime\": \"2026-05-13T21:09:22.905598885Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"timed out after 29s waiting for resolution of desired version from '4.21.15' cluster version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:160]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000c94560\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-gjf5t9\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-fg6lv9\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f247cdd-5812-4761-b99d-f988d2cb1453\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5f247cdd-5812-4761-b99d-f988d2cb1453\",\n      \"name\": \"5f247cdd-5812-4761-b99d-f988d2cb1453\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-05-13T21:07:58.388882897Z\",\n      \"endTime\": \"2026-05-13T21:09:22.905598885Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"timed out after 29s waiting for resolution of desired version from '4.21.15' cluster version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:160]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000dac400\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-mtqjkn\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-n4zbgv\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/182072e0-9b74-4cef-b677-89a453c15273\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/182072e0-9b74-4cef-b677-89a453c15273\",\n      \"name\": \"182072e0-9b74-4cef-b677-89a453c15273\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-05-13T16:20:21.870546037Z\",\n      \"endTime\": \"2026-05-13T16:22:13.927370791Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"timed out after 29s waiting for resolution of desired version from '4.21.15' cluster version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:160]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000dac400\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-mtqjkn\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-n4zbgv\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/182072e0-9b74-4cef-b677-89a453c15273\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/182072e0-9b74-4cef-b677-89a453c15273\",\n      \"name\": \"182072e0-9b74-4cef-b677-89a453c15273\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-05-13T16:20:21.870546037Z\",\n      \"endTime\": \"2026-05-13T16:22:13.927370791Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"timed out after 29s waiting for resolution of desired version from '4.21.15' cluster version\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054702578100342784","occurred_at":"2026-05-13T23:17:23Z","signature_id":"2f8145ea538c7de4b98a8174c0d95a0da3aedb0050a28975c0695c87517e07fb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054651356416839680","occurred_at":"2026-05-13T19:53:51Z","signature_id":"711787586e5347cc6f943ff07b2b74dffe57c9394b8194ceeafb912bbc81034d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054581570538311680","occurred_at":"2026-05-13T15:16:27Z","signature_id":"34da2dfccb65d727b8d34d8767ac31674d5e678ab80b997f59798757c54726cb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054503829403078656","occurred_at":"2026-05-13T10:07:37Z","signature_id":"e28b956523b2056e0b2f53bbe0ee07a446e34759829f038b829020c52590d339","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054464536823468032","occurred_at":"2026-05-13T07:31:30Z","signature_id":"1084eb940a55d1ffea9cfc06cd48d1beb2ffd3b6ca6fbf04bd7cfc5fc775395f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054413450670510080","occurred_at":"2026-05-13T04:03:36Z","signature_id":"62bfcaa315458c3c4c8ff53d329e499df17073326eb5b839ebed84e22094642d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"6d8a28d48cc4970d0f2235caa51b21154be46f90f637aecb0006fe72ad7a4ea5","failure_pattern":"DeleteResourceGroup timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-bpc899, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":9.67741935483871,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,2,1,0,0,0,0],"trend_range":"2026-05-10..2026-05-17 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:143]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00115e660\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-fbqwxv, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:143]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00115e660\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-fbqwxv, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:143]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d58210\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-bpc899, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:143]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d58210\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-bpc899, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:143]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013a6240\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-ds2snt, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:143]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013a6240\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-ds2snt, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054702578100342784","occurred_at":"2026-05-13T23:17:23Z","signature_id":"28e82b38fb6eb2b679482bb38434e6e7435e386f50e15694ed8dd1944ec139bb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054157437681799168","occurred_at":"2026-05-12T11:11:12Z","signature_id":"238cad006b640ed13097f78a67fc39f46b7cc7210094c379e1aca4a52bb86eb8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054038477145640960","occurred_at":"2026-05-12T03:18:29Z","signature_id":"03c5b2cee8fde2932a03cdd83bac6fbc7aeac7388391cf3bb53299463f3513b0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"18d5af65735558751d0c56d3c20458f218b143c5bf541874b01f142b2c1ce20c","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":6.451612903225806,"anchor_occurrences":3,"after_last_push_count":4,"trend_counts":[0,0,0,0,3,0,0,0],"trend_range":"2026-05-10..2026-05-17 UTC","prior_windows_present":1,"prior_window_starts":["2026-05-04"],"prior_runs_affected":3,"prior_last_seen_at":"2026-05-06T05:43:00Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:163]: Timed out after 2400.004s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00090e720\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.13) failed: no version in clusterversion/version status.history is greater than initial \"4.21.13\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:163]: Timed out after 2400.004s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00090e720\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.13) failed: no version in clusterversion/version status.history is greater than initial \"4.21.13\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:163]: Timed out after 2400.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0012f7218\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.21) failed: no version in clusterversion/version status.history is greater than initial \"4.20.21\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:163]: Timed out after 2400.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0012f7218\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.21) failed: no version in clusterversion/version status.history is greater than initial \"4.20.21\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:163]: Timed out after 2400.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000e0a288\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.13) failed: no version in clusterversion/version status.history is greater than initial \"4.21.13\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:163]: Timed out after 2400.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000e0a288\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.13) failed: no version in clusterversion/version status.history is greater than initial \"4.21.13\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2055037502590291968","occurred_at":"2026-05-14T21:28:15Z","signature_id":"6b810b7f01f9b04310e2db7e7862ce3e70c5b2479200c675a19a907fe1e7bad0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2055037502590291968","occurred_at":"2026-05-14T21:28:15Z","signature_id":"90878feb2d1afe5f4d0bb5fab73e07b38350d8835a19206e879e348345500678","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054979965346123776","occurred_at":"2026-05-14T17:39:38Z","signature_id":"e0b1cf1b0b7e195c4b541c8d35b95447b7d3bbc0036d0d9895c54674c475acf4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"98e32667c3e390e96a18d1f5c467888c22b014b872e3aaaa2bfc4f15efac6c45","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":6.451612903225806,"anchor_occurrences":2,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,1,1,0],"trend_range":"2026-05-10..2026-05-17 UTC","prior_windows_present":4,"prior_window_starts":["2026-04-13","2026-04-20","2026-04-27","2026-05-04"],"prior_runs_affected":6,"prior_last_seen_at":"2026-05-05T16:34:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000d401a0\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-klv5d.apps.aro.o3q8p3t2k8a4o8d.ut95.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000d401a0\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-klv5d.apps.aro.o3q8p3t2k8a4o8d.ut95.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000a969c0\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-4lwp9.apps.aro.b3n7c6n1o0c1r2h.8oqk.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000a969c0\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-4lwp9.apps.aro.b3n7c6n1o0c1r2h.8oqk.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2055636416578195456","occurred_at":"2026-05-16T13:08:08Z","signature_id":"ce37e829e61fc2c369431e223cf063d262cd2e6ac47fc5a7b0aab29f8d872b59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2055090328905453568","occurred_at":"2026-05-15T00:58:10Z","signature_id":"d0c2ac3bcc6d7ca0be01d4fb45e9eb08d211172d46f0da3b1dd2dcc427afe9dc","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"69aae224cabe21be84c0dd3ac4bc79e923f9267588a9cbcaf129d8dd2355d688","failure_pattern":"Get \"\u003curl\u003e\": net/http: TLS handshake timeout","search_query":"Get \"https://api.b4x0o7a4b6a8h9a.r5f8.uksouth.aroapp-hcp.azure-test.net:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.225806451612903,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,2,0,0,0],"trend_range":"2026-05-10..2026-05-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:164]: Unexpected error:\n    \u003c*url.Error | 0xc0007ae3f0\u003e: \n    Get \"https://api.i5k4i0d7h6e2k7l.fyje.uksouth.aroapp-hcp.azure-test.net:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:164]: Unexpected error:\n    \u003c*url.Error | 0xc0007ae3f0\u003e: \n    Get \"https://api.i5k4i0d7h6e2k7l.fyje.uksouth.aroapp-hcp.azure-test.net:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:164]: Unexpected error:\n    \u003c*url.Error | 0xc0010bb830\u003e: \n    Get \"https://api.b4x0o7a4b6a8h9a.r5f8.uksouth.aroapp-hcp.azure-test.net:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:164]: Unexpected error:\n    \u003c*url.Error | 0xc0010bb830\u003e: \n    Get \"https://api.b4x0o7a4b6a8h9a.r5f8.uksouth.aroapp-hcp.azure-test.net:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054979965346123776","occurred_at":"2026-05-14T17:39:38Z","signature_id":"95245f23db0ca430405e4902992177d319923cd533524e82ccfbda892cc9781b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054979965346123776","occurred_at":"2026-05-14T17:39:38Z","signature_id":"ae9ebf75e37a5bfff6a77a37607a58bbae258223d35e69952a51be4bd4f44ffa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"4416272c4ec2593bdc59af9eddd3d1b8fa2f2b304befb799ba37906ba3f6fdb8","failure_pattern":"Unauthorized","search_query":"Unauthorized","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.225806451612903,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-05-10..2026-05-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:222]: Timed out after 300.000s.\nExpected success, but got an error:\n    \u003c*errors.StatusError | 0xc000deac80\u003e: \n    Unauthorized\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:222]: Timed out after 300.000s.\nExpected success, but got an error:\n    \u003c*errors.StatusError | 0xc000deac80\u003e: \n    Unauthorized\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054338636807147520","occurred_at":"2026-05-12T23:10:19Z","signature_id":"b6a4224737c25b3acacf1802be83551c5789936618ac9d57a3014a381c6fc744","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"88fb289639ac585d41e102922a6ea4f479e947e9962ca681abe481db2908b42c","failure_pattern":"context deadline exceeded","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.225806451612903,"anchor_occurrences":1,"after_last_push_count":9,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-05-10..2026-05-17 UTC","prior_windows_present":2,"prior_window_starts":["2026-04-20","2026-05-04"],"prior_runs_affected":9,"prior_last_seen_at":"2026-05-07T06:18:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:139]: Unexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:139]: Unexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054702578100342784","occurred_at":"2026-05-13T23:17:23Z","signature_id":"b15c55d32190fb3d32f16f58a52f080055a52cddf394a6cd7e062c8467f788e4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"bed574ad97045960d75c75aa64c7f7747ddcebb9625b6104fadd5684b1f696d4","failure_pattern":"couldn't get current server API group list: the server has asked for the client to provide credentials","search_query":"couldn't get current server API group list: the server has asked for the client to provide credentials","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.225806451612903,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-05-10..2026-05-17 UTC","prior_windows_present":1,"prior_window_starts":["2026-05-04"],"prior_runs_affected":2,"prior_last_seen_at":"2026-05-08T00:19:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:362]: Timed out after 300.000s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:347 with:\nkubectl should execute successfully from authorized VM with OIDC token\nUnexpected error:\n    \u003c*errors.errorString | 0xc000bef620\u003e: \n    E0514 00:08:31.255742    2644 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0514 00:08:31.268033    2644 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0514 00:08:31.294823    2644 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0514 00:08:31.307820    2644 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0514 00:08:31.324327    2644 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    error: You must be logged in to the server (the server has asked for the client to provide credentials)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:362]: Timed out after 300.000s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:347 with:\nkubectl should execute successfully from authorized VM with OIDC token\nUnexpected error:\n    \u003c*errors.errorString | 0xc000bef620\u003e: \n    E0514 00:08:31.255742    2644 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0514 00:08:31.268033    2644 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0514 00:08:31.294823    2644 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0514 00:08:31.307820    2644 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    E0514 00:08:31.324327    2644 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n    error: You must be logged in to the server (the server has asked for the client to provide credentials)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054702578100342784","occurred_at":"2026-05-13T23:17:23Z","signature_id":"4b6c8b4d7b8ced44a7844df94600fc6572e60be695bc5afc98d9515e5f24cb90","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"bd7947ad546a8ce90d0eae10dd401fca344113b258b2f178687d819f18bbde79","failure_pattern":"the server was unable to return a response in the time allotted, but may still be processing the request (get clusterversions.config.openshift.io version)","search_query":"occurred","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":3.225806451612903,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-05-10..2026-05-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:164]: Unexpected error:\n    \u003c*errors.StatusError | 0xc000ebe1e0\u003e: \n    the server was unable to return a response in the time allotted, but may still be processing the request (get clusterversions.config.openshift.io version)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:164]: Unexpected error:\n    \u003c*errors.StatusError | 0xc000ebe1e0\u003e: \n    the server was unable to return a response in the time allotted, but may still be processing the request (get clusterversions.config.openshift.io version)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2055037502590291968","occurred_at":"2026-05-14T21:28:15Z","signature_id":"4fbed466282b3d4d54db8cc86e40261915a0fc22daa069aa3410fb9b677d25fc","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}