{"meta":{"start_date":"2026-06-08","end_date":"2026-06-08","start_at":"2026-06-08T00:00:00Z","end_at":"2026-06-09T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T10:07:35Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":5,"failed_runs":4,"raw_occurrences":55,"matched_occurrences":55,"runs_affected":4},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-rhbss4, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-rhbss4 in resource group rg-candidate-4-21-9l2tgc, error: failed waiting for cluster=\"cluster-candidate-4-21-rhbss4\" in resourcegroup=\"rg-candidate-4-21-9l2tgc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-rhbss4 in resource group rg-candidate-4-21-9l2tgc, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":44,"runs_affected":2,"failed_runs":2,"run_impact_percent":40,"anchor_occurrences":44,"after_last_push_count":44,"trend_counts":[0,0,0,0,0,0,44],"trend_range":"2026-06-02..2026-06-08 UTC","prior_windows_present":4,"prior_window_starts":["2026-05-11","2026-05-18","2026-05-25","2026-06-01"],"prior_runs_affected":19,"prior_last_seen_at":"2026-06-07T23:00:01Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should update NodeDrainTimeoutMinutes and upgrade a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"MISE Routing routes to the correct frontend based on version header default route returns no version header","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:266]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010468d0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-lmjs6c, error: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-lmjs6c\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-lmjs6c, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:266]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010468d0\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-lmjs6c, error: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-lmjs6c\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster shoebox-hcp-cluster in resource group shoebox-cluster-lmjs6c, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:132]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009be090\u003e: \n    failed waiting for cluster=\"mise-routing\" in resourcegroup=\"mise-v2-smoke-gchwj4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster mise-routing in resource group mise-v2-smoke-gchwj4, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:132]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009be090\u003e: \n    failed waiting for cluster=\"mise-routing\" in resourcegroup=\"mise-v2-smoke-gchwj4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster mise-routing in resource group mise-v2-smoke-gchwj4, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-7kbv2c/cluster-nightly-5-0-75csg9 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000fd68d0\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-75csg9\" in resourcegroup=\"rg-nightly-5-0-7kbv2c\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-75csg9 in resource group rg-nightly-5-0-7kbv2c, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-7kbv2c/cluster-nightly-5-0-75csg9 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000fd68d0\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-75csg9\" in resourcegroup=\"rg-nightly-5-0-7kbv2c\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-75csg9 in resource group rg-nightly-5-0-7kbv2c, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"09f4e1b48bc031d19351131e3e9b3df597ffc75ec2e5b66378713795fc5f043f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"0a43ec94c2863ca4b336e006a07732fd3d758263ca1578573e54289c13024dcf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"0f0b2273a62179badd010b1df62dd7c85496b3f8f360139ee518f3dcc98a174b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"0faa39eea1270eaacbb84c30a499b820ae55f4d4f7b2f39e33f59e66d7539eb2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"125dedfb4ee9e3f40cf6b114dc5d0bd9fb1230f0761d7ecc971bdd2b1674e675","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"130fa1abc1b5a8cbeae58cfa8192278529e9e8795fcf62f7a3d955aee93bdd00","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"17621182fd3b24f17582322146e1838ad86b8432cf9a49b499dcb7b623ee720c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"1873eb0f9ba761d8b65f56e33bcf9de75f9048bd51b6c936f7f67726715f6945","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"21756ac398d3c503a1224a8b873d8ac58cedea31e26923e3cdf74c048ef2d93e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"241d8004d5042a0026f15c89269557c079962e5c94942969e727217d8b3a0c41","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"2f9bdd4293107dcaa61f0f086fd6a06ab261ea167a379173cd99012e90001d68","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"3090fbb9e37d80f1cbf3b2a437dc92351a24782ebf8dde235b091f2cd99421bd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"321764716e31ffb8f3f6c01d36f0a67a31e392930cdba48703352f7eebd131d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"3c3ca9e4fda1bc8b1e86f4b06d740d99a058b43b278f132d4ab5925468380e90","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"446375a84add091eeecdfc12c718df4acb6f115bef0f36fba67ef0acc6bf5bd0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"46026eb3ac3364eb6aa0b3996428d9c4d3554a3ad2fbaadf2e2285deb3b231fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"4b2f058544ca14868606f51609f6fef8f5572338f401f75e9436b22ab08478c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"4ba755dd26b3dc39c5b47b2f091d879e3ee3f3d43c40ef83eee4cab1db541b39","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"66ee54e80c8954878060892a25be6de3864a84faf840e991ab7ad235bef1bb12","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"6d565c3487e3481e963a54c779d2621812477ddaa62fcf1a836169e460a8169c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"77dc8479e9555fa5d6d8b7f5acbdb786f3a575d0c609aaab59de3e786aece826","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"7a03a9e19d1fd0ca62657150a184e12239e47c4342893638485480b86a39c2c8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"7b5c597cfd9b7ee43effb532e86d6ff29e36f85568b657290d271081bb5ecba6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"7d857a4672f434e0826dbdd7a494d5225c28ae51035fa8f383ac34455f7052cb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"7e271c1515c62e413d844e050a37f98955f577ff985e2ef7aafaf058f29fc2dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"859799d54d29acd9325355cdd3a3711041d5f7ca05e61962acfe88e77c64857d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"870e74cc4343b36e78230dcebeaa90bf4de523ccbeaf208cb64900037c47d11f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"88987000f6c1d2fa346b4c9c0b037506f79301f0ec78736f2417ddb09eabc231","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"971db7ea227c9a843878caceb2fcfa6cc7714dcd32a794e05d8e355e52c7e69d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"99b448bc1ed34b0e32e08562a411076ceec822699944a95cad0492a75407dc6d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"9f586bde8dd0de828eb804e800a7dff51a26fea417c0f1dccffe66a637183d88","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"a917ccb2e5bc00ea452ee727d73a748610ca7bb7c74f6735bcb4a4255593abb4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"b5cacdffb75f6b7ea888c02abd1da06439ddb932ed6a12cdb5a177a8806a506c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"befc86fb2a918b94c0262a6b25ce70da13f55c8fb4754e2bf2b19b701e019c02","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"c4c6cc3c197588f7bb29589c0462773918f3698806f5820c9875c28d7896c8bb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"d064d8706f355aa4fea64b571976314c7a9ea1b4459e985b33c5c7e81ae48fe2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"dae1578f91a74e69e8030fcafec5cdf14e43177612c38d8e804cbf41baf7a237","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"dd6903498fb35b3984f15c5672d06bfefca7e6dd394d7e62800b47b24957e6e6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"f09e7a89c94ffe9f43cab158ec073a518a0e283fb8bf247d539dcd51dc52dd2e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"f0d4e3a6bc96382512a921178a4ab47e36e2bde9e77b2f655fa77da2e6435c9a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"f92407dc7fbb618d15793af587793243e1dd824326271d6a672734d51a7f92f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"fbbd5854e9e6d88ddba740f5a915720ffc9602f8eba76d92759a707c6ffb74a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"ff7aedc5b9001e78493983b2db42c42c3c57c696cde4d41ada952621e7242106","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064015992266166272","occurred_at":"2026-06-08T16:05:29Z","signature_id":"b2d6dfed2103db4882893237e6207e97d7a89341fc5cedde1e01486a10d47b5a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"90e9ab5a508c69fa469c74ea9a9d435aee1bda7d3791c7e7ad6b5d426a15064f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":2,"after_last_push_count":11,"trend_counts":[0,1,2,2,2,2,2],"trend_range":"2026-06-02..2026-06-08 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-01"],"prior_runs_affected":6,"prior_last_seen_at":"2026-06-07T23:00:01Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-msdg4x at version 4.22.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f7aa80\u003e: \n    failed to create HCP cluster np-skip-minor-msdg4x: failed starting cluster creation \"np-skip-minor-msdg4x\" in resourcegroup=\"rg-np-skip-minor-msdg4x-77vm7v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-msdg4x-77vm7v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-msdg4x\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.22.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-msdg4x at version 4.22.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f7aa80\u003e: \n    failed to create HCP cluster np-skip-minor-msdg4x: failed starting cluster creation \"np-skip-minor-msdg4x\" in resourcegroup=\"rg-np-skip-minor-msdg4x-77vm7v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-msdg4x-77vm7v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-msdg4x\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.22.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:704]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aa92e0\u003e: \n    failed to create HCP cluster np-dg-minor-cqhmp9: failed starting cluster creation \"np-dg-minor-cqhmp9\" in resourcegroup=\"rg-np-dg-minor-cqhmp9-ksrc76\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-cqhmp9-ksrc76/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-cqhmp9\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.21.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:704]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aa92e0\u003e: \n    failed to create HCP cluster np-dg-minor-cqhmp9: failed starting cluster creation \"np-dg-minor-cqhmp9\" in resourcegroup=\"rg-np-dg-minor-cqhmp9-ksrc76\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-cqhmp9-ksrc76/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-cqhmp9\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.21.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2064120297979645952","occurred_at":"2026-06-08T23:00:03Z","signature_id":"2db1076b47fd748669944c115001496d718af27cb3fcae3d20fdf5f78215b2e0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2064120297979645952","occurred_at":"2026-06-08T23:00:03Z","signature_id":"5ad1a1e5d0d677cdd57b4dabd203630ba60598ce0aa81af91eda783f7be8f3fc","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":2,"after_last_push_count":8,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-06-02..2026-06-08 UTC","prior_windows_present":2,"prior_window_starts":["2026-05-18","2026-05-25"],"prior_runs_affected":6,"prior_last_seen_at":"2026-05-30T02:00:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:39]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:39]: Interrupted by User","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:207]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:207]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dcad60eb8db626db10bfb0812a18e7c392049987a81c4fcbcdc5140e0032dc41","failure_pattern":"Cluster should become ready within '20.000000' minutes","search_query":"Cluster should become ready within '20.000000' minutes","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-06-02..2026-06-08 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 1200.004s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 1200.004s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"7653ffd4e02735b234dd652c0dd32332c9566363ef7806113eb84037d3ada7a9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":6,"trend_counts":[0,0,0,1,0,0,1],"trend_range":"2026-06-02..2026-06-08 UTC","prior_windows_present":3,"prior_window_starts":["2026-05-11","2026-05-18","2026-06-01"],"prior_runs_affected":5,"prior_last_seen_at":"2026-06-05T02:00:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["ime: 2h29m59.889s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:119\n        In [It] (Node Runtime: 2h29m59.889s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:119\n          At [By Step] creating the HCP cluster (Step Runtime: 8m56.845s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:151\n\n          Spec Goroutine\n          goroutine 105 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x91df528, 0xc0008d6410}, 0x8632aa5?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x921ee80, {0x91df528, 0xc0008d6410}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait20240610({0x91df598?, 0xc000262230?}, {{0x91f9240?, 0xc000bdb500?}, 0xc000df0ea0?}, 0xc00008f708, {0xc0016944f8, 0x11}, {0x8653140, 0x12}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:906\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam20240610(_, {_, _}, {{_, _}, _}, {_, _}, {{0xc001696890, 0x7}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:353\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func16.2.1({0x7f7694417860, 0xc000d14f30})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:152\n                | \n                | By(\"creating the HCP cluster\")\n                \u003e err = tc.CreateHCPClusterFromParam20240610(\n                | \tctx,\n                | \tGinkgoLogr,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x91f9850?, 0xc000d14f30?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 103\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:159 @ 06/08/26 21:51:59.012\n  \"ts\"=\"2026-06-08 21:51:59.012501\" \"level\"=0 \"msg\"=\"===== TEST CASE ENDED: FAILURE =====\"\n  \"ts\"=\"2026-06-08 21:51:59.012523\" \"level\"=0 \"msg\"=\"===== CLEANUP BEGAN =====\"\n  \"ts\"=\"2026-06-08 21:51:59.012604\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-06-08 21:51:59.012661\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-patch-tags-cluster-xt75mv\" \"resourceGroup\"=\"patch-tags-k2sn7j\"\n  \"ts\"=\"2026-06-08 21:51:59.121922\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-patch-tags-cluster-xt75mv\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-06-08 21:51:59.945884\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-4117c8518f426e648f352d29f774d8baa91734601e145475be95e6bea8053dce.yaml.gz\"\n  \"ts\"=\"2026-06-08 21:51:59.945984\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-4117c8518f426e648f352d29f774d8baa91734601e145475be95e6bea8053dce.yaml.gz\"\n  \"ts\"=\"2026-06-08 21:51:59.946058\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-06-08 21:51:59.947757\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"patch-tags-k2sn7j\"\n  \"ts\"=\"2026-06-08 21:51:59.947774\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-146\"\n  \"ts\"=\"2026-06-08 21:52:00.263378\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"patch-tags-k2sn7j\" \"deployment\"=\"pooledNonMsiScopedAssignments-fklo75htpxzxc\"\n  \"ts\"=\"2026-06-08 21:52:00.317886\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"patch-tags-k2sn7j\" \"deployment\"=\"customer-infra-patch-tags-cluster-xt75mv\"\n  \"ts\"=\"2026-06-08 21:52:01.555256\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-146\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-06-08 21:52:01.845940\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-146\" \"deployment\"=\"msi-deployment-146\"\n  \"ts\"=\"2026-06-08 21:52:01.955263\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-06-08 21:52:02.138048\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-06-08 21:52:02.138229\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"patch-tags-k2sn7j\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nime: 2h29m59.889s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:119\n        In [It] (Node Runtime: 2h29m59.889s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:119\n          At [By Step] creating the HCP cluster (Step Runtime: 8m56.845s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:151\n\n          Spec Goroutine\n          goroutine 105 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x91df528, 0xc0008d6410}, 0x8632aa5?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x921ee80, {0x91df528, 0xc0008d6410}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait20240610({0x91df598?, 0xc000262230?}, {{0x91f9240?, 0xc000bdb500?}, 0xc000df0ea0?}, 0xc00008f708, {0xc0016944f8, 0x11}, {0x8653140, 0x12}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:906\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam20240610(_, {_, _}, {{_, _}, _}, {_, _}, {{0xc001696890, 0x7}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:353\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func16.2.1({0x7f7694417860, 0xc000d14f30})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:152\n                | \n                | By(\"creating the HCP cluster\")\n                \u003e err = tc.CreateHCPClusterFromParam20240610(\n                | \tctx,\n                | \tGinkgoLogr,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x91f9850?, 0xc000d14f30?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 103\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:159 @ 06/08/26 21:51:59.012\n  \"ts\"=\"2026-06-08 21:51:59.012501\" \"level\"=0 \"msg\"=\"===== TEST CASE ENDED: FAILURE =====\"\n  \"ts\"=\"2026-06-08 21:51:59.012523\" \"level\"=0 \"msg\"=\"===== CLEANUP BEGAN =====\"\n  \"ts\"=\"2026-06-08 21:51:59.012604\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-06-08 21:51:59.012661\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-patch-tags-cluster-xt75mv\" \"resourceGroup\"=\"patch-tags-k2sn7j\"\n  \"ts\"=\"2026-06-08 21:51:59.121922\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-patch-tags-cluster-xt75mv\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-06-08 21:51:59.945884\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-4117c8518f426e648f352d29f774d8baa91734601e145475be95e6bea8053dce.yaml.gz\"\n  \"ts\"=\"2026-06-08 21:51:59.945984\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-4117c8518f426e648f352d29f774d8baa91734601e145475be95e6bea8053dce.yaml.gz\"\n  \"ts\"=\"2026-06-08 21:51:59.946058\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-06-08 21:51:59.947757\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"patch-tags-k2sn7j\"\n  \"ts\"=\"2026-06-08 21:51:59.947774\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-146\"\n  \"ts\"=\"2026-06-08 21:52:00.263378\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"patch-tags-k2sn7j\" \"deployment\"=\"pooledNonMsiScopedAssignments-fklo75htpxzxc\"\n  \"ts\"=\"2026-06-08 21:52:00.317886\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"patch-tags-k2sn7j\" \"deployment\"=\"customer-infra-patch-tags-cluster-xt75mv\"\n  \"ts\"=\"2026-06-08 21:52:01.555256\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-146\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-06-08 21:52:01.845940\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-146\" \"deployment\"=\"msi-deployment-146\"\n  \"ts\"=\"2026-06-08 21:52:01.955263\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-06-08 21:52:02.138048\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-06-08 21:52:02.138229\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"patch-tags-k2sn7j\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"0229965880d4859fafdf34c4b45eca9f3e33ce9ca472554f5ac79c449c7ebf38","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"53fa6318769bb2a786556b3fac950c44b85ab8ba4f6fa60b2949a759bcc916c1","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.19.33\": cannot downgrade from version 4.21.18 to 4.19.33; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[0,0,0,1,1,1,1],"trend_range":"2026-06-02..2026-06-08 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-01"],"prior_runs_affected":4,"prior_last_seen_at":"2026-06-07T02:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00039e480\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-mkwzwg\" resourcegroup=\"rg-np-dg-minor-mkwzwg-pcf6kp\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-mkwzwg-pcf6kp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-mkwzwg/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": invalid node pool version 4.19.33: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": cannot downgrade from version 4.21.18 to 4.19.33\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00039e480\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-mkwzwg\" resourcegroup=\"rg-np-dg-minor-mkwzwg-pcf6kp\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-mkwzwg-pcf6kp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-mkwzwg/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": invalid node pool version 4.19.33: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": cannot downgrade from version 4.21.18 to 4.19.33\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2063803203840905216","occurred_at":"2026-06-08T02:00:02Z","signature_id":"ff9c5b843f68b5e12d95ab85b0d2bf1a7da55153e66799fcd28883700873be12","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d81a996cdf791decc51216fb7be72b9db7c9b98c36c458e851bda1b4a4b600a6","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.21.17\": cannot downgrade from version 4.21.18 to 4.21.17; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":6,"trend_counts":[0,1,1,1,1,1,1],"trend_range":"2026-06-02..2026-06-08 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-01"],"prior_runs_affected":6,"prior_last_seen_at":"2026-06-07T02:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000134780\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-z52pvc\" resourcegroup=\"rg-np-downgrade-z52pvc-w5x8s7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-z52pvc-w5x8s7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-z52pvc/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": invalid node pool version 4.21.17: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": cannot downgrade from version 4.21.18 to 4.21.17\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000134780\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-z52pvc\" resourcegroup=\"rg-np-downgrade-z52pvc-w5x8s7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-z52pvc-w5x8s7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-z52pvc/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": invalid node pool version 4.21.17: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": cannot downgrade from version 4.21.18 to 4.21.17\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2063803203840905216","occurred_at":"2026-06-08T02:00:02Z","signature_id":"4e20a613518a8ebe401b69555164476e8a73a8384289308330ccf2025c327983","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"471f22e5967d4c6d260358df80d5f1c22cdca60ff8d89437ff00486c66a994f2","failure_pattern":"cluster has not yet reached Succeeded state","search_query":"cluster has not yet reached Succeeded state","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-06-02..2026-06-08 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:199]: Timed out after 2520.009s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:197 with:\ncluster has not yet reached Succeeded state\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nto equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:199]: Timed out after 2520.009s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:197 with:\ncluster has not yet reached Succeeded state\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nto equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"1b9ffb2925264c314070ba914a055d20ace8066b99c7b24cfdf7af8ad940f632","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6f962bc17a3105c00a9b0bd7f4b2aca4444ece86f9a14faacfeee24cd27ebd87","failure_pattern":"failed to wait for first cluster \"\u003ccluster\u003e\" to complete creation","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,1,0,0,0,0,1],"trend_range":"2026-06-02..2026-06-08 UTC","prior_windows_present":2,"prior_window_starts":["2026-05-25","2026-06-01"],"prior_runs_affected":3,"prior_last_seen_at":"2026-06-03T02:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"3e3a3d4f62c5e8a64ff8b5cb02b93498aaf8a6753c80e2b6b8b3864addabc2a3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[0,0,1,0,0,1,1],"trend_range":"2026-06-02..2026-06-08 UTC","prior_windows_present":3,"prior_window_starts":["2026-05-11","2026-05-18","2026-06-01"],"prior_runs_affected":7,"prior_last_seen_at":"2026-06-07T02:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:205]: failed to verify simple web app runs on cluster \"cluster-nightly-5-0-49vmww\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0000d5080\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-h7tkh.apps.aro.g5x8x9n2r7b0d4z.mce0.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:205]: failed to verify simple web app runs on cluster \"cluster-nightly-5-0-49vmww\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0000d5080\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-h7tkh.apps.aro.g5x8x9n2r7b0d4z.mce0.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2064120297979645952","occurred_at":"2026-06-08T23:00:03Z","signature_id":"970d1553d83adb9b2c3ff9515db6e1558cd0b193b0df5b377bcf4e22b01a7423","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}