{"meta":{"start_date":"2026-05-14","end_date":"2026-05-14","start_at":"2026-05-14T00:00:00Z","end_at":"2026-05-15T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T14:27:17Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":5,"failed_runs":3,"raw_occurrences":29,"matched_occurrences":29,"runs_affected":3},"rows":[{"environment":"int","failure_pattern_id":"0ece011306dab74cd22dd19f1f3c89bbdb930a2321b61be59e4a6a90aae6e3eb","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-8ldb78, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-8ldb78\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-8ldb78, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":21,"runs_affected":2,"failed_runs":2,"run_impact_percent":40,"anchor_occurrences":21,"after_last_push_count":21,"trend_counts":[0,0,0,0,0,0,21],"trend_range":"2026-05-08..2026-05-14 UTC","prior_windows_present":4,"prior_window_starts":["2026-04-13","2026-04-20","2026-04-27","2026-05-04"],"prior_runs_affected":23,"prior_last_seen_at":"2026-05-08T18:38:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes vm type NC4asT4v3 in a single cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013269f0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-8ldb78, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-8ldb78\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-8ldb78, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013269f0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-8ldb78, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-8ldb78\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-8ldb78, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000bcd1d0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-6r4k9b, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-6r4k9b in resource group rg-zstream-upgrade-4-21-s6x9zc, error: failed waiting for cluster=\"cluster-zstream-4-21-6r4k9b\" in resourcegroup=\"rg-zstream-upgrade-4-21-s6x9zc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-6r4k9b in resource group rg-zstream-upgrade-4-21-s6x9zc, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000bcd1d0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-6r4k9b, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-6r4k9b in resource group rg-zstream-upgrade-4-21-s6x9zc, error: failed waiting for cluster=\"cluster-zstream-4-21-6r4k9b\" in resourcegroup=\"rg-zstream-upgrade-4-21-s6x9zc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-6r4k9b in resource group rg-zstream-upgrade-4-21-s6x9zc, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:130]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016a43c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-t9cqjk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-t9cqjk in resource group rg-np-version-upgrade-t9cqjk-n82ftb, error: failed waiting for cluster=\"np-version-upgrade-cluster-t9cqjk\" in resourcegroup=\"rg-np-version-upgrade-t9cqjk-n82ftb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-t9cqjk in resource group rg-np-version-upgrade-t9cqjk-n82ftb, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:130]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016a43c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-t9cqjk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-t9cqjk in resource group rg-np-version-upgrade-t9cqjk-n82ftb, error: failed waiting for cluster=\"np-version-upgrade-cluster-t9cqjk\" in resourcegroup=\"rg-np-version-upgrade-t9cqjk-n82ftb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-t9cqjk in resource group rg-np-version-upgrade-t9cqjk-n82ftb, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2055037502590291968","occurred_at":"2026-05-14T21:28:15Z","signature_id":"9eaa8d74f13b4f3ace070f266d1cb1bdfa71deee5e82f3b7c4517e97b2f5c458","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"06a5f88867c35d1df0e300b3790a87bea5f8884b2f0d8e7f39983e486a23a582","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"1d0b062351c390420bd6a7fbd3214056756d79ebe52a736086015527f9c0e375","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"38d0f7d434d3d5dfd37a81e737b938af182d3115b9d4c71c3ad5b9163dfe9d0c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"5633c5035b8548ae6d6d461391a9381107d902e092f93688b08e5ed3e4ed6ab8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"574b84be02a3288a74904fdcff082b8ede1a49310701ad5d3ba313dd5cde8396","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"6582d0e2cf2ba35fbffae0a9e9d10c6945ca98894f93460c3a4e1c1330fc903a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"7825c3614613367a3b7f656dafeb2c17321ddaa8a9dd1bb0927fb72c1732f006","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"815116a8eee64aec5fcff8d4df53b68e237d0ad64a22ccce6add780a72e683d3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"83e5f252c405dca0c65f4bb154b7a79b894c7d4724f7bda1f34b6a1a699cf8e6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"8bcef61ead2dcd67687e827a45fcb0ca56ec19ac9c444dd8aac48f9455872833","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"a1ef1e1a2d7c4f7b6bc4a6cd4e8dbf3dd47e0d82c08e0245649f0bda618b6d53","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"a3d746c244e71b86329ad75ff8570b5e6fd6d24c8048f4f6a3e500287f308935","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"b0fb94ea60dee3450350024732cd741c7c6609e591db7ba19fe547a00a5cb16d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"bab1425ae35fc894918ca5de241735c2f0cf8740bf9c50345c9782fd862cbf0e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"bd5213e85cb223e04092498ba721a3512a918f669a143a292a469afae7de48b7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"c26bd7ee8e74a65f7251a41b8c73a636db54b96f1be61ecbe2808a5fe6d1550e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"caeab39c48e3f17ed84da9b7657a6023fe8406757a609bd0efe9004c32a7ca6a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"d35985e5ba5ec1c816fcedda2bb4a9943d421651986b78954b068bde1d19d9b9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"ebbcf63133d3d9fb7481757e7db2280ce10a3d20d5a84df72fb86aab84bae2cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054851787764011008","occurred_at":"2026-05-14T09:10:18Z","signature_id":"f097f69c3a95f439357b736ab90312941eebfd2235b81a5307dbfe05ca6a774d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"18d5af65735558751d0c56d3c20458f218b143c5bf541874b01f142b2c1ce20c","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":40,"anchor_occurrences":3,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-05-08..2026-05-14 UTC","prior_windows_present":1,"prior_window_starts":["2026-05-04"],"prior_runs_affected":3,"prior_last_seen_at":"2026-05-06T05:43:00Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:163]: Timed out after 2400.004s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00090e720\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.13) failed: no version in clusterversion/version status.history is greater than initial \"4.21.13\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:163]: Timed out after 2400.004s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00090e720\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.13) failed: no version in clusterversion/version status.history is greater than initial \"4.21.13\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:163]: Timed out after 2400.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0012f7218\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.21) failed: no version in clusterversion/version status.history is greater than initial \"4.20.21\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:163]: Timed out after 2400.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0012f7218\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.21) failed: no version in clusterversion/version status.history is greater than initial \"4.20.21\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:163]: Timed out after 2400.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000e0a288\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.13) failed: no version in clusterversion/version status.history is greater than initial \"4.21.13\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:163]: Timed out after 2400.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000e0a288\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.13) failed: no version in clusterversion/version status.history is greater than initial \"4.21.13\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2055037502590291968","occurred_at":"2026-05-14T21:28:15Z","signature_id":"6b810b7f01f9b04310e2db7e7862ce3e70c5b2479200c675a19a907fe1e7bad0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2055037502590291968","occurred_at":"2026-05-14T21:28:15Z","signature_id":"90878feb2d1afe5f4d0bb5fab73e07b38350d8835a19206e879e348345500678","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054979965346123776","occurred_at":"2026-05-14T17:39:38Z","signature_id":"e0b1cf1b0b7e195c4b541c8d35b95447b7d3bbc0036d0d9895c54674c475acf4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"52d9258f03a60774b49eaa70a514574e25b40d0eeb6ddfa13f252e24164f6be2","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-jfrqxb\" in resourcegroup=\"rg-zstream-upgrade-4-20-wwwdbx\" to finish deleting: context canceled","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":40,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-05-08..2026-05-14 UTC","prior_windows_present":0,"prior_runs_affected":8,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00171a558\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-cj2npq\" in resourcegroup=\"rg-np-version-upgrade-cj2npq-b8gwnd\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00171a558\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-cj2npq\" in resourcegroup=\"rg-np-version-upgrade-cj2npq-b8gwnd\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001c08360\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-jfrqxb\" in resourcegroup=\"rg-zstream-upgrade-4-20-wwwdbx\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001c08360\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-jfrqxb\" in resourcegroup=\"rg-zstream-upgrade-4-20-wwwdbx\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2055037502590291968","occurred_at":"2026-05-14T21:28:15Z","signature_id":"5202052add0b795a81215c941dd88792de0c42f4ea36362d9fb0693e22b0e015","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054979965346123776","occurred_at":"2026-05-14T17:39:38Z","signature_id":"e4cae0392febcb2e2061ff00bab566e51107ecbeed46bf72ab7c22551560b7ea","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"69aae224cabe21be84c0dd3ac4bc79e923f9267588a9cbcaf129d8dd2355d688","failure_pattern":"Get \"\u003curl\u003e\": net/http: TLS handshake timeout","search_query":"Get \"https://api.b4x0o7a4b6a8h9a.r5f8.uksouth.aroapp-hcp.azure-test.net:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-05-08..2026-05-14 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:164]: Unexpected error:\n    \u003c*url.Error | 0xc0007ae3f0\u003e: \n    Get \"https://api.i5k4i0d7h6e2k7l.fyje.uksouth.aroapp-hcp.azure-test.net:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:164]: Unexpected error:\n    \u003c*url.Error | 0xc0007ae3f0\u003e: \n    Get \"https://api.i5k4i0d7h6e2k7l.fyje.uksouth.aroapp-hcp.azure-test.net:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:164]: Unexpected error:\n    \u003c*url.Error | 0xc0010bb830\u003e: \n    Get \"https://api.b4x0o7a4b6a8h9a.r5f8.uksouth.aroapp-hcp.azure-test.net:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:164]: Unexpected error:\n    \u003c*url.Error | 0xc0010bb830\u003e: \n    Get \"https://api.b4x0o7a4b6a8h9a.r5f8.uksouth.aroapp-hcp.azure-test.net:443/apis/config.openshift.io/v1/clusterversions/version\": net/http: TLS handshake timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054979965346123776","occurred_at":"2026-05-14T17:39:38Z","signature_id":"95245f23db0ca430405e4902992177d319923cd533524e82ccfbda892cc9781b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2054979965346123776","occurred_at":"2026-05-14T17:39:38Z","signature_id":"ae9ebf75e37a5bfff6a77a37607a58bbae258223d35e69952a51be4bd4f44ffa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"bd7947ad546a8ce90d0eae10dd401fca344113b258b2f178687d819f18bbde79","failure_pattern":"the server was unable to return a response in the time allotted, but may still be processing the request (get clusterversions.config.openshift.io version)","search_query":"occurred","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-05-08..2026-05-14 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:164]: Unexpected error:\n    \u003c*errors.StatusError | 0xc000ebe1e0\u003e: \n    the server was unable to return a response in the time allotted, but may still be processing the request (get clusterversions.config.openshift.io version)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:164]: Unexpected error:\n    \u003c*errors.StatusError | 0xc000ebe1e0\u003e: \n    the server was unable to return a response in the time allotted, but may still be processing the request (get clusterversions.config.openshift.io version)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2055037502590291968","occurred_at":"2026-05-14T21:28:15Z","signature_id":"4fbed466282b3d4d54db8cc86e40261915a0fc22daa069aa3410fb9b677d25fc","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}