{"meta":{"start_date":"2026-06-08","end_date":"2026-06-14","start_at":"2026-06-08T00:00:00Z","end_at":"2026-06-15T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T07:43:15Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":44,"failed_runs":39,"raw_occurrences":262,"matched_occurrences":261,"runs_affected":38},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-pvzm27, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-pvzm27 in resource group rg-candidate-4-21-kzhdgd, error: failed waiting for cluster=\"cluster-candidate-4-21-pvzm27\" in resourcegroup=\"rg-candidate-4-21-kzhdgd\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-pvzm27 in resource group rg-candidate-4-21-kzhdgd, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":169,"runs_affected":23,"failed_runs":23,"run_impact_percent":52.27272727272727,"anchor_occurrences":169,"after_last_push_count":169,"trend_counts":[0,44,23,3,5,82,4,8],"trend_range":"2026-06-07..2026-06-14 UTC","prior_windows_present":4,"prior_window_starts":["2026-05-11","2026-05-18","2026-05-25","2026-06-01"],"prior_runs_affected":40,"prior_last_seen_at":"2026-06-07T23:00:01Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":8},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":11},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":6},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should update NodeDrainTimeoutMinutes and upgrade a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"MISE Routing routes to the correct frontend based on version header default route returns no version header","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001512570\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-tn428m, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-tn428m\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-tn428m, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001512570\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-tn428m, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-tn428m\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-tn428m, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-v2vrn5/cluster-nightly-4-23-qz5lb5 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001756060\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-qz5lb5\" in resourcegroup=\"rg-nightly-4-23-v2vrn5\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-qz5lb5 in resource group rg-nightly-4-23-v2vrn5, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-v2vrn5/cluster-nightly-4-23-qz5lb5 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001756060\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-qz5lb5\" in resourcegroup=\"rg-nightly-4-23-v2vrn5\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-qz5lb5 in resource group rg-nightly-4-23-v2vrn5, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:137]: HCP cluster rg-nightly-4-20-tvmzpn/cluster-nightly-4-20-rp6bj2 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000eb12c0\u003e: \n    failed to create HCP cluster cluster-nightly-4-20-rp6bj2, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-20-rp6bj2 in resource group rg-nightly-4-20-tvmzpn, error: failed waiting for cluster=\"cluster-nightly-4-20-rp6bj2\" in resourcegroup=\"rg-nightly-4-20-tvmzpn\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-20-rp6bj2 in resource group rg-nightly-4-20-tvmzpn, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:137]: HCP cluster rg-nightly-4-20-tvmzpn/cluster-nightly-4-20-rp6bj2 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000eb12c0\u003e: \n    failed to create HCP cluster cluster-nightly-4-20-rp6bj2, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-20-rp6bj2 in resource group rg-nightly-4-20-tvmzpn, error: failed waiting for cluster=\"cluster-nightly-4-20-rp6bj2\" in resourcegroup=\"rg-nightly-4-20-tvmzpn\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-20-rp6bj2 in resource group rg-nightly-4-20-tvmzpn, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2066294654927638528","occurred_at":"2026-06-14T23:00:10Z","signature_id":"6e96334feb91b7f90d33318a3fc176e7afa7381ce89bdaa16af16ec5c53b71b7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2066294654927638528","occurred_at":"2026-06-14T23:00:10Z","signature_id":"d244b112e2ad26fa052da477855393886ad64eac64a5a30d56680dd7cbbd2fbd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2066294654927638528","occurred_at":"2026-06-14T23:00:10Z","signature_id":"e9559b96d780291ed2600bbf1a17c839e4b520c0d89bc853aac303456b63337f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2066294654927638528","occurred_at":"2026-06-14T23:00:10Z","signature_id":"f5176a51e014b4f1934ae61b69a680947d8b6e67b0b7049c6bf365ea41fdaecc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065977561912971264","occurred_at":"2026-06-14T02:00:09Z","signature_id":"0e2afedaca1d94638dff6d2a04031de42b90ab4a7820b7d9858643cdddfe5a06","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065977561912971264","occurred_at":"2026-06-14T02:00:09Z","signature_id":"77483405d2bf1d4a9ab2349f1d536f8e85935ef41bd2be81172eeba69582c8dc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065977561912971264","occurred_at":"2026-06-14T02:00:09Z","signature_id":"f84e126cbdeafcc40249d2408ff1a4c9bfd90b3f8862180dc9d58d8f9f9431f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065977561912971264","occurred_at":"2026-06-14T02:00:09Z","signature_id":"fcb787bfb2f18eedb859f9dda0df8a69a6fb27fa69cf4a8c94783d8b6261df73","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065932263165530112","occurred_at":"2026-06-13T23:00:09Z","signature_id":"0efd7a11cb057b0aec96ab5bfbd67f9bed5d8540f42648f4d7182063d9538390","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065932263165530112","occurred_at":"2026-06-13T23:00:09Z","signature_id":"9ee000acddd0fba7399e9c0f0083b44860c3f29bc291b509a910dd1343206a75","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065615170868088832","occurred_at":"2026-06-13T02:00:08Z","signature_id":"4edf6fee90ed5858139054c814d406c16d5131fbc5ca8b622581771deb7e2c42","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065615170868088832","occurred_at":"2026-06-13T02:00:08Z","signature_id":"e8111712700769ba5ea4edd004a453254ff50e4d7e2a573f0974c341b5572219","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065569872280031232","occurred_at":"2026-06-12T23:00:08Z","signature_id":"1ba30c8ec52b3a78614a8be7fc4a6f2b9a8505f1072f8e99347c508739b01832","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065569872280031232","occurred_at":"2026-06-12T23:00:08Z","signature_id":"4ae94c9df0b86bc0853a36e2d61459f4031ba967983e5437424753d4f8aa7c3c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"02d8e750b79439a02a0144b0d55d1d8e45e0894ec30e2cd60f7a51a37051da98","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"091eae7736ff11889af2b2df6097ca0453bbe1c4616f159cc466136cc176214d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"0985dc74fb0146b797d6c177c5c96f1c382f1b0eca6dc5d1b8cbe3fb71e2eb9a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"0b7d8231c3d2a72fee7ae3fc5d41eab3da69893791a142c4157d8a40cb93d630","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"0d18bc5b5ec708937139322becaa5776c4bcceff3a0b385256ffa835ce367ec5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"100205b6955aacca60b32f6018c517190a6156257bed7ad1a0e2fc1ed90de4d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"13ef952f83f21679e60013e4663a9b8b9f99a59f6c376bfd83badba273a737e2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"154b51b7b0ab9d7bfc7e2a09ffde8332426211b97c4a30b54ea73829f9f12336","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"1adc961c272f0de9acb16d62331bd2c1867eb1036c563cc7b009bfdd36cb72b0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"20a333384fad8637a91a7a8dd6fa75792aaafb18a31aaffc41f96f756c077b17","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"21159667ff630148e565cd2b104ff5c11ffea56ce7eca154eb86eabbef8e61d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"23de6018809533d7b894609515c213ff2764a7ef40dd2d6d3a8b8999dd170bef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"354a182204d334c15a1698aa279b283f295e0110abe9895c70125f17c5f30714","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"38231908bb31ae762e3bb21bb13e6cb85c2996b42e0e55fd618ab4947f6c7f4c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"3aaea858048f7c94a30f01a59c010b88c5623ef13e2e01c1c837b2bb01088315","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"405210bca6c42487be29e42461771aa0df4a3e35eb978188609d4342f60bf8ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"4106bfe74631bedab0d390fb412456c5285dbf70b890c354f7bba686c04d8488","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"439ab8efa1fdd64d0144a50c68916766374a807fc1a3326475669e45b68728c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"4d82a6644e8b655f8a093e987a0e9c8d1ec3b5b6a0fe008cd05e8e79daffdc4e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"53b4f1c3dc2b8d1f9f268c50d7f31f4cf10656d8e8a2fc897e9608c25a5006fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"53f16bc29a3dcdc3b771b2cda5fd701377fe089a234f96b09ead20cd84f1f56b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"5a61fcb8d405cb6b8dd8adc528e261797451c097455161fd261fb9a65c4994fe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"5b23d9329edaedc47f7c83678cab9cb13b97f1dc7ad2466822e75643d7b5e185","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"5c862a16f4c3d98a03edff9d52f010bd85f94d14b5a9ee240ba1fd49608b221f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"7c30ccfc2501eeb57ad0d5daeec6bd7e3fd3282134bd3f63dcaca929265e5e9e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"7cc9b2e6fb08f197f396b2c993f630f75244988372b45baae52170755fcba744","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"8425278efb5a9e9f4c47c12b59dfde1d02c3415fec084bb673b5cf60e4379ab2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"881fb6a265648d2a2f457729dadd908b86c6b7912840642087ac440b55b32907","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"8d4d19cc833cb773c4d95b163617d10f840fd30140242ac4f1d747e6e30a6e50","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"8ebe5c7651992dbe7ff78adec3cde986716638a811ea466befc660d1102da685","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"930a5e054cc504007ba2c1ca5cfc45d418cb3acfdaf8fbf490938ccff502ed71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"99fe3414aa9c6c6d6c00268cae0b7816768a90df0efc53823a57191f0a3020f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"b658661f3c2062a3d3e2b22239369ce9cce19d3a728b2809f8b07a7c4540bd70","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"bf51da0b2c3a1887607e2043eaabd12768e3ad70fcbbf32338a0629d0cfe02d7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"d24d66660e25c54a0361dd6cb2bf224fd6770d26340cab7088446a3f6760d6a7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"d373c89cacb08f33d7f816638bb81b65635a36b72196bb6b8ecc9faad17e2772","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"d8cce136b78c51b77f9bef66701dd0d6acb2917469e9bf42dc5c4fbbb3152b32","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"de82d70a2ab81a9dc29718440053944dd92b50ecd927ea783b8802554f892d88","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"e1851db0318d470aaf7cadd4ca9b6f0fcd796477deec3174cc7f0b9d1b53780b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"081eb13762f019c798144698bd940558c27ec3d9883376f7868d15710fc12173","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"0d1201473d80b8f1184f394bba1203bf0a6119b21a740eb31698c083c59b0d83","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"0f04c7a671ed4b38a7b84815bafd8806e2c2d71ec3d1f0682a14ba834122f20d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"1bc2931f0b08962a427e63cf87749a4ba2984769a65fcb0b887c528908c3ea5f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"1c9e682d5309beeb9626b59d8fc3c19b82acf85cd7b8d20dd47a7a26a4a9add8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"30bb0d7028f80fb18fc100bf5cba4d5c9020490350bd57ebc89aafa6beb60818","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"32aceb4d859f50761bfc9e55d1489d8bd8c2ac9427aeaaf162fbf6d226534d66","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"38fdc9f8214be89992804aa603330cb6fe9fa1ecdaf9f072cd9287070c6cfb0b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"3ba5477bd7fb3b69a4f2bd15f86df95d9bedfc83fa1275909ab4af5d444579ae","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"43ee4f15e6f8ff06e0d8aafd2d3fa23921d63f08dd7ddd0bf40880335e7c49ba","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"4a7ada8586277880e376f0ee17fc0cd1d14f734d9c581209fb24a7f9bfeea4ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"4c15eef89717ad4085fdf4bf5da62f7227617a66d8f07dc632034caef3406569","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"4c58a89c1c53aa0867a02a3f99f33e45a7bf1b2bd014771fb040d1c856594e34","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"57bfa3060ac0914f81717b9b9423b23d381eed61f55b4162b6f9118285ebb3cb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"7ca2d18467236b837547b98ad0099f57ffcbf6e374534a5f26768130d34ff74e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"7d71b077c32beb8cfc5ff53131e0ecf36a02d4577aa1074cc8469790052aeafd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"80608d7b7a0326af77c86127893febcf4760848b03d8af7f739e25d8b6c7c2a2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"84b15fc89c76df18d95d344599b9d0394a3c9310b3d12e04ac4625d82c0bfdff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"86218b3910d6f910992aab8a9aea689c936c7b963097b1c490bf60815c628544","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"87d25e0c10440141933109650af2e4f14c453269e34af01ae11d5771cd6ca867","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"8d2edfd73dbcb7548a1109b5cd7e17e8480d4ae7409e8a77c6845badc4825587","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"93ab5d1f68614ed2c4622f637b5947806ce1a4a3ba1c69445db7637feee6ebc3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"99fcfe1885e74594168a77268c8297847f8e51a94ec7d6a5560f611686675f86","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"9ef7199db34b22176f999bf7153d7b87f04cdc4534f615ead6fae2e33b30d0c6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"a3473e0ee74a735950a2273663a670c8c71df5f569035984ace11ce456808ba3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"b8d8da3a79b9a804f33bb50f7b79662fa1b3b26f9c558bd259693798d5410ea5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"b981155f42a33c43fcfcc33b78a82c93942232eb74134c7c571e40630ead8b10","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"be05d2fb86fd2fa8a2725539948e5b33dc714906247e5ae97695cc9820b9abcb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"bf259989f9359e7039ce71809f838f9f528fa891add195b395f6f55b5109cc14","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"bf639703dab04cdc292d0fe260d3ed6adf04758fbacc530e335ddb37b245319e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"c309d9ffd70a22263116a61d31716fc991c8fa728598abd1b4d573db8bc814d1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"cb2dc64109886c45b7eaaa47c085b83187619cdc335a62fd639a712824d0ff37","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"cea3b70f5824caf1fa23764f3f08ffa6d80f2d1f5329eabdd0c28f998ccc209a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"cefdb0571b80ec4dbbe75d37485e9f5fd4a54ecff8d0bfcb70d3e3787a8a6943","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"d8c8e8fd406b64b718349e554a112591b6bda616177cc090aff939bb5641ccf3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"d9e87306d9f6cdafd24bdad5f054b512f8f99626bf3c8a4eb93ab465c46b1124","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"e97bf0bcc43168e4366f2880a307be6e90948adb7c08181dfba8aad3bad4baa9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"ebdae0484e3ad536f02a921cb001ee082aaac65567177922369bfa616ace32ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"fd99e47981c3c6fd357e0032deb6db7a5093ec7bfe5c266a894751fbea191ded","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065252825906024448","occurred_at":"2026-06-12T02:00:02Z","signature_id":"08b61e80c5ccfdfc37bbb7373f3a13f4de99d2cd42c7b309d33d94ef317ac5f0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065252825906024448","occurred_at":"2026-06-12T02:00:02Z","signature_id":"1c84fee9f9bb64463b516123e66b71e1bd4c9d8c1fd7e1a9f211eb195ea94c00","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065207457541525504","occurred_at":"2026-06-11T23:00:02Z","signature_id":"8c84792dc7fc683c4bc2e2b7121cd7aa8a85319227bab7844b1708bff445d875","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065207457541525504","occurred_at":"2026-06-11T23:00:02Z","signature_id":"cc537cd5eaacd194e2cfb4e930b3faa948a61ddfe8751351edbb2ff400fcb2c0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065133461022183424","occurred_at":"2026-06-11T18:06:00Z","signature_id":"897ea65756daec9faa905677c938f13528c412af6902d75e6affb822376462e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065133461022183424","occurred_at":"2026-06-11T18:06:00Z","signature_id":"af1dfca2123bd3256e8df77d8db24de16ca4e7421a2d964520820b0febd56690","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064999166156738560","occurred_at":"2026-06-11T09:12:21Z","signature_id":"1f562dd8dfe058b5cb72dc24c089d783b528b54c85a0581c4dc141522b3d8d07","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064790437436067840","occurred_at":"2026-06-10T19:22:56Z","signature_id":"e42d0d536bea2a2825881c2fbbdb14a1c2f3092e0a1a999ec38114dea2b89b69","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064600993957416960","occurred_at":"2026-06-10T06:42:17Z","signature_id":"7769e21e8e0e25a3b28a4d26f7c03e221fa63c03e88a09e545dbbcde73dd6d18","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064601088287313920","occurred_at":"2026-06-10T06:41:04Z","signature_id":"4af98909b7cadfcd7669a6b3b2910e09cfebe2ee3b42b291d4a227cdbbd8fc4e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064425150199107584","occurred_at":"2026-06-09T19:11:25Z","signature_id":"7e509204663de1041322d583434342eaadafda17d51e7eeda1a39e99962f3e92","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064396799321313280","occurred_at":"2026-06-09T17:18:46Z","signature_id":"595ea2a0b0a16cae02d995131ef64eb25872da4fa709c4c4d7214dc5710cdb52","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064396799321313280","occurred_at":"2026-06-09T17:18:46Z","signature_id":"a55b64c48f3248d8ce4b52cb22e6d907463e0ca467c2a061f63cd84d751b39a7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064396319065116672","occurred_at":"2026-06-09T17:16:51Z","signature_id":"349f1814f2668e5b71ebf02b93d024e87eb5a3d72775bf25646ae00f9397c20e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064396319065116672","occurred_at":"2026-06-09T17:16:51Z","signature_id":"38de80d8706b44c08c01b2a8c363d15110ae600984ec9536a1b9d24218e36fdb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064396319065116672","occurred_at":"2026-06-09T17:16:51Z","signature_id":"c40934efb8a0c36d54d9ac7fc3aa5478fd79b13a78f23812cb1fc060ac6db64c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064310885882531840","occurred_at":"2026-06-09T11:37:22Z","signature_id":"3705190eecfbba1c9ea51d9eec1551324cdc8eb274b9bde829ead5b6966e2e3a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064310885882531840","occurred_at":"2026-06-09T11:37:22Z","signature_id":"5515548644808dca6ae2c15348ba82a72b566ec2132d29fabad81f437d546dfe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064310885882531840","occurred_at":"2026-06-09T11:37:22Z","signature_id":"844a0be22d6f0db0efb47416f9afc1192a2df91b608c90b252fa8fcc2525c42b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064310885882531840","occurred_at":"2026-06-09T11:37:22Z","signature_id":"928a47cb7944bb323bec31fa2a5eb2975c526028b36598f2c149cce344dc83a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064310885882531840","occurred_at":"2026-06-09T11:37:22Z","signature_id":"a96e9bb6a3c1c06e294ca56e88c2ee62c54caa2ef50ccd1e23b8595b7e5e7d09","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064279856266350592","occurred_at":"2026-06-09T09:34:04Z","signature_id":"88910cff0ca3e97103f1cf61ad68d8d3d30060e5c912607a89e67adbc2ff2314","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064264482309279744","occurred_at":"2026-06-09T08:32:59Z","signature_id":"9a3c9f282540cd44f49cf8c739dc0fa1b8fe77a908fb12e831a3b72f463c3da7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","occurred_at":"2026-06-09T02:00:03Z","signature_id":"4b2fdb0b9028b543839c0319632f4da5cc1da42c264cf1ea145a9c68e3211b72","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","occurred_at":"2026-06-09T02:00:03Z","signature_id":"7a19e46774a7fcfb8f26f0a8ca344c40ed042d97c7b5b00d009abeb7e1f1fe79","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","occurred_at":"2026-06-09T02:00:03Z","signature_id":"86519bea76cee7f46f7264e61e683906b531a249469acbbb7fa8b6400176ea5d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","occurred_at":"2026-06-09T02:00:03Z","signature_id":"a0304bca149abe4d70e3e12d31cba0a08934172e0298275e0c3c321a78013adf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","occurred_at":"2026-06-09T02:00:03Z","signature_id":"abd0fc63fd2db9f860e7eb6fe246b2d00a034d5b3c3244a18aad6858b101bf25","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","occurred_at":"2026-06-09T02:00:03Z","signature_id":"c7a5e8940abe777af1144339954157b679d83e1470af1aaa49f44ae6d47592b2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","occurred_at":"2026-06-09T02:00:03Z","signature_id":"ca41a13db418271cbeff80f8bede1b8af93828c5394f8d69b3f67de6c998632d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","occurred_at":"2026-06-09T02:00:03Z","signature_id":"cff2360eac2cad168884885eaabbbf173a7621bebc74e33d785976a60f7c9db8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","occurred_at":"2026-06-09T02:00:03Z","signature_id":"d090d8b072b52dae181e782adaae6dce5b5171c994ae4865ad600637c423eeea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","occurred_at":"2026-06-09T02:00:03Z","signature_id":"f3e57f3875d065bdf55d7525d1966201997ac0ac1a15a5e3a3d5bceeae51438f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"09f4e1b48bc031d19351131e3e9b3df597ffc75ec2e5b66378713795fc5f043f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"0a43ec94c2863ca4b336e006a07732fd3d758263ca1578573e54289c13024dcf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"0f0b2273a62179badd010b1df62dd7c85496b3f8f360139ee518f3dcc98a174b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"0faa39eea1270eaacbb84c30a499b820ae55f4d4f7b2f39e33f59e66d7539eb2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"125dedfb4ee9e3f40cf6b114dc5d0bd9fb1230f0761d7ecc971bdd2b1674e675","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"130fa1abc1b5a8cbeae58cfa8192278529e9e8795fcf62f7a3d955aee93bdd00","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"17621182fd3b24f17582322146e1838ad86b8432cf9a49b499dcb7b623ee720c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"1873eb0f9ba761d8b65f56e33bcf9de75f9048bd51b6c936f7f67726715f6945","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"21756ac398d3c503a1224a8b873d8ac58cedea31e26923e3cdf74c048ef2d93e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"241d8004d5042a0026f15c89269557c079962e5c94942969e727217d8b3a0c41","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"2f9bdd4293107dcaa61f0f086fd6a06ab261ea167a379173cd99012e90001d68","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"3090fbb9e37d80f1cbf3b2a437dc92351a24782ebf8dde235b091f2cd99421bd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"321764716e31ffb8f3f6c01d36f0a67a31e392930cdba48703352f7eebd131d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"3c3ca9e4fda1bc8b1e86f4b06d740d99a058b43b278f132d4ab5925468380e90","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"446375a84add091eeecdfc12c718df4acb6f115bef0f36fba67ef0acc6bf5bd0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"46026eb3ac3364eb6aa0b3996428d9c4d3554a3ad2fbaadf2e2285deb3b231fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"4b2f058544ca14868606f51609f6fef8f5572338f401f75e9436b22ab08478c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"4ba755dd26b3dc39c5b47b2f091d879e3ee3f3d43c40ef83eee4cab1db541b39","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"66ee54e80c8954878060892a25be6de3864a84faf840e991ab7ad235bef1bb12","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"6d565c3487e3481e963a54c779d2621812477ddaa62fcf1a836169e460a8169c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"77dc8479e9555fa5d6d8b7f5acbdb786f3a575d0c609aaab59de3e786aece826","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"7a03a9e19d1fd0ca62657150a184e12239e47c4342893638485480b86a39c2c8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"7b5c597cfd9b7ee43effb532e86d6ff29e36f85568b657290d271081bb5ecba6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"7d857a4672f434e0826dbdd7a494d5225c28ae51035fa8f383ac34455f7052cb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"7e271c1515c62e413d844e050a37f98955f577ff985e2ef7aafaf058f29fc2dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"859799d54d29acd9325355cdd3a3711041d5f7ca05e61962acfe88e77c64857d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"870e74cc4343b36e78230dcebeaa90bf4de523ccbeaf208cb64900037c47d11f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"88987000f6c1d2fa346b4c9c0b037506f79301f0ec78736f2417ddb09eabc231","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"971db7ea227c9a843878caceb2fcfa6cc7714dcd32a794e05d8e355e52c7e69d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"99b448bc1ed34b0e32e08562a411076ceec822699944a95cad0492a75407dc6d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"9f586bde8dd0de828eb804e800a7dff51a26fea417c0f1dccffe66a637183d88","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"a917ccb2e5bc00ea452ee727d73a748610ca7bb7c74f6735bcb4a4255593abb4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"b5cacdffb75f6b7ea888c02abd1da06439ddb932ed6a12cdb5a177a8806a506c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"befc86fb2a918b94c0262a6b25ce70da13f55c8fb4754e2bf2b19b701e019c02","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"c4c6cc3c197588f7bb29589c0462773918f3698806f5820c9875c28d7896c8bb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"d064d8706f355aa4fea64b571976314c7a9ea1b4459e985b33c5c7e81ae48fe2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"dae1578f91a74e69e8030fcafec5cdf14e43177612c38d8e804cbf41baf7a237","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"dd6903498fb35b3984f15c5672d06bfefca7e6dd394d7e62800b47b24957e6e6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"f09e7a89c94ffe9f43cab158ec073a518a0e283fb8bf247d539dcd51dc52dd2e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"f0d4e3a6bc96382512a921178a4ab47e36e2bde9e77b2f655fa77da2e6435c9a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"f92407dc7fbb618d15793af587793243e1dd824326271d6a672734d51a7f92f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"fbbd5854e9e6d88ddba740f5a915720ffc9602f8eba76d92759a707c6ffb74a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"ff7aedc5b9001e78493983b2db42c42c3c57c696cde4d41ada952621e7242106","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064015992266166272","occurred_at":"2026-06-08T16:05:29Z","signature_id":"b2d6dfed2103db4882893237e6207e97d7a89341fc5cedde1e01486a10d47b5a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-5nhhnb, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-5nhhnb\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":35,"runs_affected":13,"failed_runs":13,"run_impact_percent":29.545454545454547,"anchor_occurrences":35,"after_last_push_count":35,"trend_counts":[0,0,3,2,4,10,8,8],"trend_range":"2026-06-07..2026-06-14 UTC","prior_windows_present":3,"prior_window_starts":["2026-05-11","2026-05-18","2026-05-25"],"prior_runs_affected":16,"prior_last_seen_at":"2026-05-27T23:14:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-w5pt4m\" for cluster \"cluster-nightly-4-21-w5pt4m\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000976450\u003e: \n    failed to create NodePool np-w5pt4m, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-w5pt4m in resource group rg-nightly-4-21-x7zgfd, error: failed waiting for nodepool=\"np-w5pt4m\" for cluster \"cluster-nightly-4-21-w5pt4m\" in resourcegroup=\"rg-nightly-4-21-x7zgfd\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-w5pt4m\" for cluster \"cluster-nightly-4-21-w5pt4m\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000976450\u003e: \n    failed to create NodePool np-w5pt4m, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-w5pt4m in resource group rg-nightly-4-21-x7zgfd, error: failed waiting for nodepool=\"np-w5pt4m\" for cluster \"cluster-nightly-4-21-w5pt4m\" in resourcegroup=\"rg-nightly-4-21-x7zgfd\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-t2zvsb\" for cluster \"cluster-nightly-4-22-t2zvsb\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000c15050\u003e: \n    failed to create NodePool np-t2zvsb, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-t2zvsb in resource group rg-nightly-4-22-fsszt4, error: failed waiting for nodepool=\"np-t2zvsb\" for cluster \"cluster-nightly-4-22-t2zvsb\" in resourcegroup=\"rg-nightly-4-22-fsszt4\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-t2zvsb\" for cluster \"cluster-nightly-4-22-t2zvsb\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000c15050\u003e: \n    failed to create NodePool np-t2zvsb, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-t2zvsb in resource group rg-nightly-4-22-fsszt4, error: failed waiting for nodepool=\"np-t2zvsb\" for cluster \"cluster-nightly-4-22-t2zvsb\" in resourcegroup=\"rg-nightly-4-22-fsszt4\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013d4438\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-5nhhnb, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-5nhhnb\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013d4438\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-5nhhnb, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-5nhhnb\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2066294654927638528","occurred_at":"2026-06-14T23:00:10Z","signature_id":"504316a9fd986ea48d6a14af2023a507ad31f2389ecef9429227e6ed31e6feaf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2066294654927638528","occurred_at":"2026-06-14T23:00:10Z","signature_id":"800f3b127d933d04d1437020af7b18ce0f7ebcbf0d9c1e9cca214365d3e601bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2066294654927638528","occurred_at":"2026-06-14T23:00:10Z","signature_id":"a99b1d6a09accbe6835e89d1ad019bf7fbcf124ad9a72a1e3bc42ae9648adfe2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2066294654927638528","occurred_at":"2026-06-14T23:00:10Z","signature_id":"e61faa5b3559a794be8011212e4e874ec65eebcde3404aec4c65dd3b49d9a46a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065977561912971264","occurred_at":"2026-06-14T02:00:09Z","signature_id":"69ca21e400060ccd767be54483a59e9c40ae4b1ec34be8789f857a37a18b15af","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065977561912971264","occurred_at":"2026-06-14T02:00:09Z","signature_id":"ba5068312f013d9430ba740d82a2401b597eb214fb54c8d7a29c4d5c03105fea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065977561912971264","occurred_at":"2026-06-14T02:00:09Z","signature_id":"d9331bb55997ee490723bd853ae1f0bb9dc1282804f001a40adaaaeb99a1cd5d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065977561912971264","occurred_at":"2026-06-14T02:00:09Z","signature_id":"e6ef7fedce19f73d309c3f7472b02faadf28104fe9aeb762af2c2f32afef5b25","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065932263165530112","occurred_at":"2026-06-13T23:00:09Z","signature_id":"224d5d874238f1cd22488873ca7190b72187df0d9fe09841e5db2b96275a1b4e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065932263165530112","occurred_at":"2026-06-13T23:00:09Z","signature_id":"88032920372bb580b92d64fe51e4bdd8991eda298f3795aa36cd7737c90b4a69","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065932263165530112","occurred_at":"2026-06-13T23:00:09Z","signature_id":"e2d6bd557c45f7b473546eb7111383b1daa7ca88a22b4701e0ee86e877890da0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065932263165530112","occurred_at":"2026-06-13T23:00:09Z","signature_id":"f5d8698bd794108aedcde836540090a583a69ebbaa1fe948c20bc81ecc35b80e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065615170868088832","occurred_at":"2026-06-13T02:00:08Z","signature_id":"2af954d36535a0a7481035736c9a91d83a0f0ffc777925839aa2f87413518070","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065615170868088832","occurred_at":"2026-06-13T02:00:08Z","signature_id":"3bc8e0373dbd9e1f19bfd4306bfcbe51c1682a324b18c0e03ed2b11acfa65b40","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065615170868088832","occurred_at":"2026-06-13T02:00:08Z","signature_id":"b85582a724900abf00f1a29eb4d59920e3a06c0530e620bc8245baeefeeeb491","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065615170868088832","occurred_at":"2026-06-13T02:00:08Z","signature_id":"f8053c826e6cd91fead74e70a0aa2d6016090d2ff2b76651bb137a4f3a22b9e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065569872280031232","occurred_at":"2026-06-12T23:00:08Z","signature_id":"130891177f485e67bdfbe8ff60c6562fa23854ff2e9b8984c8e3076cfc755f1e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065569872280031232","occurred_at":"2026-06-12T23:00:08Z","signature_id":"38c391c5c43275e07687ecf81d9f659c10282bcfd1e7ae7392554bd2dbe83dd6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065569872280031232","occurred_at":"2026-06-12T23:00:08Z","signature_id":"3d34868016feb023d1866cad2d4c699c6bcc76c10a81e20cd3f75ff1644cadab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065569872280031232","occurred_at":"2026-06-12T23:00:08Z","signature_id":"9ad66cd155d972a703bf28d8f4848bd980d8d451fcc8edf0e09ea43ad9605d61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065569872280031232","occurred_at":"2026-06-12T23:00:08Z","signature_id":"d13184628d675826323dd26cadaa12e0a780f9045426c1684a424fad6ab109c2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065303808690360320","occurred_at":"2026-06-12T05:22:54Z","signature_id":"0278c145e3158102ddddbf7e4b92fa0c3c9dbe45ba0905b49a336f1c71510213","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065303808690360320","occurred_at":"2026-06-12T05:22:54Z","signature_id":"2607c5fb3aecfe545d2232dc82520c6d75a7331f4a971e04c92baf205de20693","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065303808690360320","occurred_at":"2026-06-12T05:22:54Z","signature_id":"27dfbed561753b0b13868865cd4e80e798040be912eb6800a37c3161138e4122","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065303808690360320","occurred_at":"2026-06-12T05:22:54Z","signature_id":"b3ca3adbf39d77a03622f25f00bcfd34e5223db45f42db3c230a8f26a76998a4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065303808690360320","occurred_at":"2026-06-12T05:22:54Z","signature_id":"f735632862021d035112daef3b3e1a2649bc0caa225202fc84ec078889dde62b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064999166156738560","occurred_at":"2026-06-11T09:12:21Z","signature_id":"2941394c649e35c40080997ff35fd327f53cce95fd8fae48411c14dcd7b70e1d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064892795172163584","occurred_at":"2026-06-11T02:09:40Z","signature_id":"35b0046c8024b6533ae4ef988d82fc15cc7483616dee9c1514a6e677dca73311","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064892795172163584","occurred_at":"2026-06-11T02:09:40Z","signature_id":"8eb6d39ad7f904f6f20fb2e538cbae5c0e47eff7450f825f0ebb13249cfbd826","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064892701127479296","occurred_at":"2026-06-11T02:09:18Z","signature_id":"85a48cb96dde09770771775b70187dd4765302c34f5c5c98a1cdd7d00c138e4b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064600993957416960","occurred_at":"2026-06-10T06:42:17Z","signature_id":"301cec170c53cf8d8b7e426b98a8b89509fc6fc1e1b6aa2e45b12321f95d1026","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064601088287313920","occurred_at":"2026-06-10T06:41:04Z","signature_id":"d29e8b84a68e64de80d4cef40e3b6928e3f1f55c7e4085b40d491859f17607ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064396319065116672","occurred_at":"2026-06-09T17:16:51Z","signature_id":"526b0473053428aa7590fc58282c30611713aa9a50937f90d7e0ae5d8dacb7ee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064396319065116672","occurred_at":"2026-06-09T17:16:51Z","signature_id":"e64ab0717150294a49d4e303bf7f7c2d51c7972d8b7f0fade88f4fa311044600","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064310885882531840","occurred_at":"2026-06-09T11:37:22Z","signature_id":"ade999842ddd3e74c5c690b7471eb7ba8f12f7bd7de77ead056f002b96d49d4f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":16,"runs_affected":10,"failed_runs":10,"run_impact_percent":22.727272727272727,"anchor_occurrences":16,"after_last_push_count":22,"trend_counts":[0,2,3,2,4,5,0,0],"trend_range":"2026-06-07..2026-06-14 UTC","prior_windows_present":2,"prior_window_starts":["2026-05-18","2026-05-25"],"prior_runs_affected":15,"prior_last_seen_at":"2026-05-30T02:00:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:30]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:30]: Interrupted by User","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:207]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:207]: Interrupted by User","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:46]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:46]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064995434417360896","occurred_at":"2026-06-11T08:57:31Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064979974896488448","occurred_at":"2026-06-11T07:56:06Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064892701127479296","occurred_at":"2026-06-11T02:09:18Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064892559024459776","occurred_at":"2026-06-11T02:08:44Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064599912179306496","occurred_at":"2026-06-10T06:43:36Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064599912179306496","occurred_at":"2026-06-10T06:43:36Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064444081047605248","occurred_at":"2026-06-09T19:39:47Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064436508076544000","occurred_at":"2026-06-09T19:29:02Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064436508076544000","occurred_at":"2026-06-09T19:29:02Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"90e9ab5a508c69fa469c74ea9a9d435aee1bda7d3791c7e7ad6b5d426a15064f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":6,"runs_affected":3,"failed_runs":3,"run_impact_percent":6.818181818181818,"anchor_occurrences":6,"after_last_push_count":15,"trend_counts":[2,2,2,2,0,0,0,0],"trend_range":"2026-06-07..2026-06-14 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-01"],"prior_runs_affected":8,"prior_last_seen_at":"2026-06-07T23:00:01Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-9kjrzr at version 4.22.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e08900\u003e: \n    failed to create HCP cluster np-skip-minor-9kjrzr: failed starting cluster creation \"np-skip-minor-9kjrzr\" in resourcegroup=\"rg-np-skip-minor-9kjrzr-7gs6rf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-9kjrzr-7gs6rf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-9kjrzr\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.22.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-9kjrzr at version 4.22.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e08900\u003e: \n    failed to create HCP cluster np-skip-minor-9kjrzr: failed starting cluster creation \"np-skip-minor-9kjrzr\" in resourcegroup=\"rg-np-skip-minor-9kjrzr-7gs6rf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-9kjrzr-7gs6rf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-9kjrzr\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.22.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:704]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b9caa0\u003e: \n    failed to create HCP cluster np-dg-minor-9c7d7p: failed starting cluster creation \"np-dg-minor-9c7d7p\" in resourcegroup=\"rg-np-dg-minor-9c7d7p-rhs2s4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-9c7d7p-rhs2s4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-9c7d7p\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.21.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:704]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b9caa0\u003e: \n    failed to create HCP cluster np-dg-minor-9c7d7p: failed starting cluster creation \"np-dg-minor-9c7d7p\" in resourcegroup=\"rg-np-dg-minor-9c7d7p-rhs2s4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-9c7d7p-rhs2s4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-9c7d7p\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.21.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:704]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001040b40\u003e: \n    failed to create HCP cluster np-dg-minor-95jzkb: failed starting cluster creation \"np-dg-minor-95jzkb\" in resourcegroup=\"rg-np-dg-minor-95jzkb-g9rpcw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-95jzkb-g9rpcw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-95jzkb\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.21.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:704]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001040b40\u003e: \n    failed to create HCP cluster np-dg-minor-95jzkb: failed starting cluster creation \"np-dg-minor-95jzkb\" in resourcegroup=\"rg-np-dg-minor-95jzkb-g9rpcw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-95jzkb-g9rpcw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-95jzkb\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.21.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2064845065389346816","occurred_at":"2026-06-10T23:00:01Z","signature_id":"3341dd64a3acbe4e116886eb801cee2e83a5a5e09ad4567ef88a2ee4f821f990","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2064845065389346816","occurred_at":"2026-06-10T23:00:01Z","signature_id":"608ceb4d0f69da6e26c29c6c19b820dd0d0a1814d4e32993c524f767d55b406b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2064482691524333568","occurred_at":"2026-06-09T23:00:04Z","signature_id":"3a3c6f7f498c1a582b3535b57f14b8795faefffe30fdd73a30f7217871698430","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2064482691524333568","occurred_at":"2026-06-09T23:00:04Z","signature_id":"6fa9bd64410cbb9e76117cebe3f64e3a4d5983b74baa2fcf9a0d73959c84ad01","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2064120297979645952","occurred_at":"2026-06-08T23:00:03Z","signature_id":"2db1076b47fd748669944c115001496d718af27cb3fcae3d20fdf5f78215b2e0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2064120297979645952","occurred_at":"2026-06-08T23:00:03Z","signature_id":"5ad1a1e5d0d677cdd57b4dabd203630ba60598ce0aa81af91eda783f7be8f3fc","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"7426c1a835b553e399edab9b5f3d89e2c53ed02e293edf09d1e41a3543dc8dec","failure_pattern":"ERROR CODE: InvalidTemplateDeployment; provider Microsoft.Compute","search_query":"ERROR CODE: InvalidTemplateDeployment","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":11.363636363636363,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,1,2,2],"trend_range":"2026-06-07..2026-06-14 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:130]: failed to deploy test VM\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d273a0\u003e: \n    failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-9xbp9m\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-9xbp9m/providers/Microsoft.Resources/deployments/test-vm\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplateDeployment\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplateDeployment\",\n        \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.Compute/virtualMachines (2023-03-01)' reported preflight validation errors. Tracking id is '21503528-9017-4784-8156-8f2140dfcfce'. See inner errors for details.\",\n        \"details\": [\n          {\n            \"code\": \"QuotaExceeded\",\n            \"message\": \"Operation could not be completed as it results in exceeding approved standardDSv5Family Cores quota. Additional details - Deployment Model: Resource Manager, Location: uksouth, Current Limit: 0, Current Usage: 0, Additional Required: 2, (Minimum) New Limit Required: 2. Setup Alerts when Quota reaches threshold. Learn more at https://aka.ms/quotamonitoringalerting . Submit a request for Quota increase at https://aka.ms/ProdportalCRP/#blade/Microsoft_Azure_Capacity/UsageAndQuota.ReactView/Parameters/%7B%22subscriptionId%22:%22XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%22,%22command%22:%22openQuotaApprovalBlade%22,%22quotas%22:[%7B%22location%22:%22uksouth%22,%22providerId%22:%22Microsoft.Compute%22,%22resourceName%22:%22standardDSv5Family%22,%22quotaRequest%22:%7B%22properties%22:%7B%22limit%22:2,%22unit%22:%22Count%22,%22name%22:%7B%22value%22:%22standardDSv5Family%22%7D%7D%7D%7D]%7D by specifying parameters listed in the ‘Details’ section for deployment to succeed. Please read more about quota limits at https://docs.microsoft.com/en-us/azure/azure-supportability/per-vm-quota-requests\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:130]: failed to deploy test VM\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d273a0\u003e: \n    failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-9xbp9m\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-9xbp9m/providers/Microsoft.Resources/deployments/test-vm\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplateDeployment\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplateDeployment\",\n        \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.Compute/virtualMachines (2023-03-01)' reported preflight validation errors. Tracking id is '21503528-9017-4784-8156-8f2140dfcfce'. See inner errors for details.\",\n        \"details\": [\n          {\n            \"code\": \"QuotaExceeded\",\n            \"message\": \"Operation could not be completed as it results in exceeding approved standardDSv5Family Cores quota. Additional details - Deployment Model: Resource Manager, Location: uksouth, Current Limit: 0, Current Usage: 0, Additional Required: 2, (Minimum) New Limit Required: 2. Setup Alerts when Quota reaches threshold. Learn more at https://aka.ms/quotamonitoringalerting . Submit a request for Quota increase at https://aka.ms/ProdportalCRP/#blade/Microsoft_Azure_Capacity/UsageAndQuota.ReactView/Parameters/%7B%22subscriptionId%22:%22XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%22,%22command%22:%22openQuotaApprovalBlade%22,%22quotas%22:[%7B%22location%22:%22uksouth%22,%22providerId%22:%22Microsoft.Compute%22,%22resourceName%22:%22standardDSv5Family%22,%22quotaRequest%22:%7B%22properties%22:%7B%22limit%22:2,%22unit%22:%22Count%22,%22name%22:%7B%22value%22:%22standardDSv5Family%22%7D%7D%7D%7D]%7D by specifying parameters listed in the ‘Details’ section for deployment to succeed. Please read more about quota limits at https://docs.microsoft.com/en-us/azure/azure-supportability/per-vm-quota-requests\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:130]: failed to deploy test VM\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0000d5ae0\u003e: \n    failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-7ktm25\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-7ktm25/providers/Microsoft.Resources/deployments/test-vm\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplateDeployment\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplateDeployment\",\n        \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.Compute/virtualMachines (2023-03-01)' reported preflight validation errors. Tracking id is 'df96e1a8-3336-4b6e-8a5f-4f6ea39c547e'. See inner errors for details.\",\n        \"details\": [\n          {\n            \"code\": \"QuotaExceeded\",\n            \"message\": \"Operation could not be completed as it results in exceeding approved standardDSv5Family Cores quota. Additional details - Deployment Model: Resource Manager, Location: uksouth, Current Limit: 0, Current Usage: 0, Additional Required: 2, (Minimum) New Limit Required: 2. Setup Alerts when Quota reaches threshold. Learn more at https://aka.ms/quotamonitoringalerting . Submit a request for Quota increase at https://aka.ms/ProdportalCRP/#blade/Microsoft_Azure_Capacity/UsageAndQuota.ReactView/Parameters/%7B%22subscriptionId%22:%22XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%22,%22command%22:%22openQuotaApprovalBlade%22,%22quotas%22:[%7B%22location%22:%22uksouth%22,%22providerId%22:%22Microsoft.Compute%22,%22resourceName%22:%22standardDSv5Family%22,%22quotaRequest%22:%7B%22properties%22:%7B%22limit%22:2,%22unit%22:%22Count%22,%22name%22:%7B%22value%22:%22standardDSv5Family%22%7D%7D%7D%7D]%7D by specifying parameters listed in the ‘Details’ section for deployment to succeed. Please read more about quota limits at https://docs.microsoft.com/en-us/azure/azure-supportability/per-vm-quota-requests\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:130]: failed to deploy test VM\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0000d5ae0\u003e: \n    failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-7ktm25\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-7ktm25/providers/Microsoft.Resources/deployments/test-vm\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplateDeployment\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplateDeployment\",\n        \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.Compute/virtualMachines (2023-03-01)' reported preflight validation errors. Tracking id is 'df96e1a8-3336-4b6e-8a5f-4f6ea39c547e'. See inner errors for details.\",\n        \"details\": [\n          {\n            \"code\": \"QuotaExceeded\",\n            \"message\": \"Operation could not be completed as it results in exceeding approved standardDSv5Family Cores quota. Additional details - Deployment Model: Resource Manager, Location: uksouth, Current Limit: 0, Current Usage: 0, Additional Required: 2, (Minimum) New Limit Required: 2. Setup Alerts when Quota reaches threshold. Learn more at https://aka.ms/quotamonitoringalerting . Submit a request for Quota increase at https://aka.ms/ProdportalCRP/#blade/Microsoft_Azure_Capacity/UsageAndQuota.ReactView/Parameters/%7B%22subscriptionId%22:%22XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%22,%22command%22:%22openQuotaApprovalBlade%22,%22quotas%22:[%7B%22location%22:%22uksouth%22,%22providerId%22:%22Microsoft.Compute%22,%22resourceName%22:%22standardDSv5Family%22,%22quotaRequest%22:%7B%22properties%22:%7B%22limit%22:2,%22unit%22:%22Count%22,%22name%22:%7B%22value%22:%22standardDSv5Family%22%7D%7D%7D%7D]%7D by specifying parameters listed in the ‘Details’ section for deployment to succeed. Please read more about quota limits at https://docs.microsoft.com/en-us/azure/azure-supportability/per-vm-quota-requests\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:130]: failed to deploy test VM\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010931e0\u003e: \n    failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-fjsk59\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-fjsk59/providers/Microsoft.Resources/deployments/test-vm\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplateDeployment\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplateDeployment\",\n        \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.Compute/virtualMachines (2023-03-01)' reported preflight validation errors. Tracking id is 'af5eb3b1-d28d-40aa-b393-7c5df027bafd'. See inner errors for details.\",\n        \"details\": [\n          {\n            \"code\": \"QuotaExceeded\",\n            \"message\": \"Operation could not be completed as it results in exceeding approved standardDSv5Family Cores quota. Additional details - Deployment Model: Resource Manager, Location: uksouth, Current Limit: 0, Current Usage: 0, Additional Required: 2, (Minimum) New Limit Required: 2. Setup Alerts when Quota reaches threshold. Learn more at https://aka.ms/quotamonitoringalerting . Submit a request for Quota increase at https://aka.ms/ProdportalCRP/#blade/Microsoft_Azure_Capacity/UsageAndQuota.ReactView/Parameters/%7B%22subscriptionId%22:%22XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%22,%22command%22:%22openQuotaApprovalBlade%22,%22quotas%22:[%7B%22location%22:%22uksouth%22,%22providerId%22:%22Microsoft.Compute%22,%22resourceName%22:%22standardDSv5Family%22,%22quotaRequest%22:%7B%22properties%22:%7B%22limit%22:2,%22unit%22:%22Count%22,%22name%22:%7B%22value%22:%22standardDSv5Family%22%7D%7D%7D%7D]%7D by specifying parameters listed in the ‘Details’ section for deployment to succeed. Please read more about quota limits at https://docs.microsoft.com/en-us/azure/azure-supportability/per-vm-quota-requests\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:130]: failed to deploy test VM\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010931e0\u003e: \n    failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-fjsk59\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-fjsk59/providers/Microsoft.Resources/deployments/test-vm\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplateDeployment\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplateDeployment\",\n        \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.Compute/virtualMachines (2023-03-01)' reported preflight validation errors. Tracking id is 'af5eb3b1-d28d-40aa-b393-7c5df027bafd'. See inner errors for details.\",\n        \"details\": [\n          {\n            \"code\": \"QuotaExceeded\",\n            \"message\": \"Operation could not be completed as it results in exceeding approved standardDSv5Family Cores quota. Additional details - Deployment Model: Resource Manager, Location: uksouth, Current Limit: 0, Current Usage: 0, Additional Required: 2, (Minimum) New Limit Required: 2. Setup Alerts when Quota reaches threshold. Learn more at https://aka.ms/quotamonitoringalerting . Submit a request for Quota increase at https://aka.ms/ProdportalCRP/#blade/Microsoft_Azure_Capacity/UsageAndQuota.ReactView/Parameters/%7B%22subscriptionId%22:%22XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%22,%22command%22:%22openQuotaApprovalBlade%22,%22quotas%22:[%7B%22location%22:%22uksouth%22,%22providerId%22:%22Microsoft.Compute%22,%22resourceName%22:%22standardDSv5Family%22,%22quotaRequest%22:%7B%22properties%22:%7B%22limit%22:2,%22unit%22:%22Count%22,%22name%22:%7B%22value%22:%22standardDSv5Family%22%7D%7D%7D%7D]%7D by specifying parameters listed in the ‘Details’ section for deployment to succeed. Please read more about quota limits at https://docs.microsoft.com/en-us/azure/azure-supportability/per-vm-quota-requests\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2066294654927638528","occurred_at":"2026-06-14T23:00:10Z","signature_id":"ad3204fc6ccf5ad28aa950ca6c5cdb36e9785a15d3b5d6844a6413e18dccf619","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065977561912971264","occurred_at":"2026-06-14T02:00:09Z","signature_id":"3eed430451de2f9abd7fb309ae03aa3bad3c24b0be4640a972fd88d000b34ace","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065932263165530112","occurred_at":"2026-06-13T23:00:09Z","signature_id":"61247fbe34626c91d73d49f2f7d5f4f639487e390dce5a64af6ad1784287a7a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065615170868088832","occurred_at":"2026-06-13T02:00:08Z","signature_id":"e86a40108734edad969cf766bf9f1f0b9ffc44eb43d51e3e502976796202076f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065569872280031232","occurred_at":"2026-06-12T23:00:08Z","signature_id":"1b5767639f3af140c9f406f1605c460243ecb68398f18c744e40c8531509944a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dcad60eb8db626db10bfb0812a18e7c392049987a81c4fcbcdc5140e0032dc41","failure_pattern":"Cluster should become ready within '20.000000' minutes","search_query":"Cluster should become ready within '20.000000' minutes","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":9.090909090909092,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,1,1,0,0,2,0,0],"trend_range":"2026-06-07..2026-06-14 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 1200.006s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 1200.006s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 1200.004s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 1200.004s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"34131747b0a1da5e284954907f00fe2548fbe113f02c09d48cdb3ae0bc323c34","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"7653ffd4e02735b234dd652c0dd32332c9566363ef7806113eb84037d3ada7a9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064310885882531840","occurred_at":"2026-06-09T11:37:22Z","signature_id":"7653ffd4e02735b234dd652c0dd32332c9566363ef7806113eb84037d3ada7a9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"7653ffd4e02735b234dd652c0dd32332c9566363ef7806113eb84037d3ada7a9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"471f22e5967d4c6d260358df80d5f1c22cdca60ff8d89437ff00486c66a994f2","failure_pattern":"cluster has not yet reached Succeeded state","search_query":"cluster has not yet reached Succeeded state","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":9.090909090909092,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,1,1,0,0,2,0,0],"trend_range":"2026-06-07..2026-06-14 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:199]: Timed out after 2520.004s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:197 with:\ncluster has not yet reached Succeeded state\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nto equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:199]: Timed out after 2520.004s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:197 with:\ncluster has not yet reached Succeeded state\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nto equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:199]: Timed out after 2520.007s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:197 with:\ncluster has not yet reached Succeeded state\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nto equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:199]: Timed out after 2520.007s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:197 with:\ncluster has not yet reached Succeeded state\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nto equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:199]: Timed out after 2520.005s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:197 with:\ncluster has not yet reached Succeeded state\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nto equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:199]: Timed out after 2520.005s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:197 with:\ncluster has not yet reached Succeeded state\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nto equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"8d91b2adc4fb8c3b24f642854a4c21503092a450afa8d61186befc6b2bc18c5a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"7d95624bb890ba417cbb3103837082359335d59a82653de667188c3d8134af7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064310885882531840","occurred_at":"2026-06-09T11:37:22Z","signature_id":"3b2c8703c3d23bcc2f6a7689b763841419941d28dedafe3f9b4b08f22e36bc47","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"1b9ffb2925264c314070ba914a055d20ace8066b99c7b24cfdf7af8ad940f632","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":6.818181818181818,"anchor_occurrences":4,"after_last_push_count":9,"trend_counts":[0,1,0,0,0,3,0,0],"trend_range":"2026-06-07..2026-06-14 UTC","prior_windows_present":3,"prior_window_starts":["2026-05-11","2026-05-18","2026-06-01"],"prior_runs_affected":7,"prior_last_seen_at":"2026-06-05T02:00:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"MISE Routing routes to the correct frontend based on version header default route returns no version header","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["077e8 sp=0xc000e077e0 pc=0x2ed3cc1\ncreated by os/signal.Notify.func1.1 in goroutine 84\n\t/usr/lib/golang/src/os/signal/signal.go:152 +0x1f\n\ngoroutine 104 gp=0xc000201180 m=nil [runnable]:\nruntime.gopark(0xc000e07f88?, 0x3?, 0x0?, 0x0?, 0xc000e07f6a?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000e07df0 sp=0xc000e07dd0 pc=0x2ecb50e\nruntime.selectgo(0xc000e07f88, 0xc000e07f64, 0x0?, 0x0, 0xc000000000?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000e07f30 sp=0xc000e07df0 pc=0x2ea62f7\ngithub.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2(0xc000efd1e0?)\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:131 +0x7d fp=0xc000e07fc8 sp=0xc000e07f30 pc=0x33174bd\ngithub.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.gowrap1()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:158 +0x24 fp=0xc000e07fe0 sp=0xc000e07fc8 pc=0x3317404\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000e07fe8 sp=0xc000e07fe0 pc=0x2ed3cc1\ncreated by github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts in goroutine 30\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:128 +0x165\n\ngoroutine 30 gp=0xc0002016c0 m=nil [select, 149 minutes]:\nruntime.gopark(0xc000fa4990?, 0x5?, 0x28?, 0x39?, 0xc000fa3cee?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000fa38f0 sp=0xc000fa38d0 pc=0x2ecb50e\nruntime.selectgo(0xc000fa4990, 0xc000fa3ce4, 0x0?, 0x0, 0x0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000fa3a30 sp=0xc000fa38f0 pc=0x2ea62f7\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode(_, {0x4b, 0x4, {0x861dc94, 0x8}, 0xc000e0fb80, {{0xa91353c, 0x5b}, 0xd0, {0x0, ...}, ...}, ...}, ...)\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:923 +0xf65 fp=0xc000fa4d28 sp=0xc000fa3a30 pc=0x33422c5\ngithub.com/onsi/ginkgo/v2/internal.(*group).attemptSpec(0xc000fa6b88, 0x1, {{0xc000ec8e08?, 0xc00070c8c0?, 0x2?}, 0x0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:199 +0xbd3 fp=0xc000fa5d48 sp=0xc000fa4d28 pc=0x3327713\ngithub.com/onsi/ginkgo/v2/internal.(*group).run(0xc000fa6b88, {0xc000a9e500?, 0x0?, 0x0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:349 +0x905 fp=0xc000fa6950 sp=0xc000fa5d48 pc=0x3329ca5\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runSpecs(0xc000a4e008, {0x0, 0x0}, {0xc81b8c0, 0x0, 0x0}, {0xc00007a094, 0x2a}, 0x0, {0xc000a9e4e0, ...})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:496 +0xa9b fp=0xc000fa6e00 sp=0xc000fa6950 pc=0x333d79b\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).RunSpec(_, {_, _}, {_, _, _}, {_, _}, {0xc00007a094, 0x2a}, ...)\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite_patch.go:68 +0x3c5 fp=0xc000fa6ea8 sp=0xc000fa6e00 pc=0x3344b05\ngithub.com/openshift-eng/openshift-tests-extension/pkg/ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite.func1.1({0x2e5aa85?, 0xc0002b0240?})\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260422120742-50dec0b5ecf3/pkg/ginkgo/util.go:84 +0x2ba fp=0xc000fa7b78 sp=0xc000fa6ea8 pc=0x6f4bb5a\ngithub.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.runSpec({0x91e1708, 0xc000e98ff0}, 0xc000efd1e0, 0x1)\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260422120742-50dec0b5ecf3/pkg/extension/extensiontests/spec.go:596 +0xee fp=0xc000fa7c20 sp=0xc000fa7b78 pc=0x6f41dce\ngithub.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260422120742-50dec0b5ecf3/pkg/extension/extensiontests/spec.go:237 +0x196 fp=0xc000fa7fe0 sp=0xc000fa7c20 pc=0x6f3db36\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000fa7fe8 sp=0xc000fa7fe0 pc=0x2ed3cc1\ncreated by github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260422120742-50dec0b5ecf3/pkg/extension/extensiontests/spec.go:230 +0x1e5\n\nrax    0xca\nrbx    0x0\nrcx    0x2ed5ae3\nrdx    0x0\nrdi    0xc7f0b70\nrsi    0x80\nrbp    0x7fffd43d8bf0\nrsp    0x7fffd43d8ba8\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x1\nr13    0x1\nr14    0xc7ee5e0\nr15    0x5\nrip    0x2ed5ae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\n077e8 sp=0xc000e077e0 pc=0x2ed3cc1\ncreated by os/signal.Notify.func1.1 in goroutine 84\n\t/usr/lib/golang/src/os/signal/signal.go:152 +0x1f\n\ngoroutine 104 gp=0xc000201180 m=nil [runnable]:\nruntime.gopark(0xc000e07f88?, 0x3?, 0x0?, 0x0?, 0xc000e07f6a?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000e07df0 sp=0xc000e07dd0 pc=0x2ecb50e\nruntime.selectgo(0xc000e07f88, 0xc000e07f64, 0x0?, 0x0, 0xc000000000?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000e07f30 sp=0xc000e07df0 pc=0x2ea62f7\ngithub.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2(0xc000efd1e0?)\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:131 +0x7d fp=0xc000e07fc8 sp=0xc000e07f30 pc=0x33174bd\ngithub.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.gowrap1()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:158 +0x24 fp=0xc000e07fe0 sp=0xc000e07fc8 pc=0x3317404\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000e07fe8 sp=0xc000e07fe0 pc=0x2ed3cc1\ncreated by github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts in goroutine 30\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:128 +0x165\n\ngoroutine 30 gp=0xc0002016c0 m=nil [select, 149 minutes]:\nruntime.gopark(0xc000fa4990?, 0x5?, 0x28?, 0x39?, 0xc000fa3cee?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000fa38f0 sp=0xc000fa38d0 pc=0x2ecb50e\nruntime.selectgo(0xc000fa4990, 0xc000fa3ce4, 0x0?, 0x0, 0x0?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000fa3a30 sp=0xc000fa38f0 pc=0x2ea62f7\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode(_, {0x4b, 0x4, {0x861dc94, 0x8}, 0xc000e0fb80, {{0xa91353c, 0x5b}, 0xd0, {0x0, ...}, ...}, ...}, ...)\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:923 +0xf65 fp=0xc000fa4d28 sp=0xc000fa3a30 pc=0x33422c5\ngithub.com/onsi/ginkgo/v2/internal.(*group).attemptSpec(0xc000fa6b88, 0x1, {{0xc000ec8e08?, 0xc00070c8c0?, 0x2?}, 0x0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:199 +0xbd3 fp=0xc000fa5d48 sp=0xc000fa4d28 pc=0x3327713\ngithub.com/onsi/ginkgo/v2/internal.(*group).run(0xc000fa6b88, {0xc000a9e500?, 0x0?, 0x0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:349 +0x905 fp=0xc000fa6950 sp=0xc000fa5d48 pc=0x3329ca5\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runSpecs(0xc000a4e008, {0x0, 0x0}, {0xc81b8c0, 0x0, 0x0}, {0xc00007a094, 0x2a}, 0x0, {0xc000a9e4e0, ...})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:496 +0xa9b fp=0xc000fa6e00 sp=0xc000fa6950 pc=0x333d79b\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).RunSpec(_, {_, _}, {_, _, _}, {_, _}, {0xc00007a094, 0x2a}, ...)\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite_patch.go:68 +0x3c5 fp=0xc000fa6ea8 sp=0xc000fa6e00 pc=0x3344b05\ngithub.com/openshift-eng/openshift-tests-extension/pkg/ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite.func1.1({0x2e5aa85?, 0xc0002b0240?})\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260422120742-50dec0b5ecf3/pkg/ginkgo/util.go:84 +0x2ba fp=0xc000fa7b78 sp=0xc000fa6ea8 pc=0x6f4bb5a\ngithub.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.runSpec({0x91e1708, 0xc000e98ff0}, 0xc000efd1e0, 0x1)\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260422120742-50dec0b5ecf3/pkg/extension/extensiontests/spec.go:596 +0xee fp=0xc000fa7c20 sp=0xc000fa7b78 pc=0x6f41dce\ngithub.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260422120742-50dec0b5ecf3/pkg/extension/extensiontests/spec.go:237 +0x196 fp=0xc000fa7fe0 sp=0xc000fa7c20 pc=0x6f3db36\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000fa7fe8 sp=0xc000fa7fe0 pc=0x2ed3cc1\ncreated by github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260422120742-50dec0b5ecf3/pkg/extension/extensiontests/spec.go:230 +0x1e5\n\nrax    0xca\nrbx    0x0\nrcx    0x2ed5ae3\nrdx    0x0\nrdi    0xc7f0b70\nrsi    0x80\nrbp    0x7fffd43d8bf0\nrsp    0x7fffd43d8ba8\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x1\nr13    0x1\nr14    0xc7ee5e0\nr15    0x5\nrip    0x2ed5ae1\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command","{0xc00007a094, 0x2a}, 0x0, {0xc0007868e0, ...})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:496\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).RunSpec(_, {_, _}, {_, _, _}, {_, _}, {0xc00007a094, 0x2a}, ...)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite_patch.go:68\n            github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite.func1.1({0x2e5aa85?, 0xc000ab0d80?})\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260422120742-50dec0b5ecf3/pkg/ginkgo/util.go:84\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.runSpec({0x91e1708, 0xc000e157c0}, 0xc000a57380, 0x1)\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260422120742-50dec0b5ecf3/pkg/extension/extensiontests/spec.go:596\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260422120742-50dec0b5ecf3/pkg/extension/extensiontests/spec.go:237\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260422120742-50dec0b5ecf3/pkg/extension/extensiontests/spec.go:230\n\n          goroutine 88 [select]\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2(0xc000a57380?)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:131\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts in goroutine 87\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:128\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:137 @ 06/12/26 13:49:03.006\n  \"ts\"=\"2026-06-12 13:49:03.007583\" \"level\"=0 \"msg\"=\"===== TEST CASE ENDED: FAILURE =====\"\n  \"ts\"=\"2026-06-12 13:49:03.007634\" \"level\"=0 \"msg\"=\"===== CLEANUP BEGAN =====\"\n  \"ts\"=\"2026-06-12 13:49:03.007761\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-06-12 13:49:03.007887\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-candidate-4-20-5pvqrv-67ffr4\" \"resourceGroup\"=\"rg-candidate-4-20-vpbqs7\"\n  \"ts\"=\"2026-06-12 13:49:03.075105\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-candidate-4-20-5pvqrv-67ffr4\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-06-12 13:49:03.778877\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-d4ee2620a775558818d65bdad60edec7ff8775b05730402bc0de88c0d1c3d86f.yaml.gz\"\n  \"ts\"=\"2026-06-12 13:49:03.779022\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-d4ee2620a775558818d65bdad60edec7ff8775b05730402bc0de88c0d1c3d86f.yaml.gz\"\n  \"ts\"=\"2026-06-12 13:49:03.779140\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-06-12 13:49:03.781368\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-candidate-4-20-vpbqs7\"\n  \"ts\"=\"2026-06-12 13:49:03.781396\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-148\"\n  \"ts\"=\"2026-06-12 13:49:04.152510\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-candidate-4-20-vpbqs7\" \"deployment\"=\"pooledNonMsiScopedAssignments-34bborbknryhu\"\n  \"ts\"=\"2026-06-12 13:49:04.166599\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-148\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-06-12 13:49:04.243967\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-candidate-4-20-vpbqs7\" \"deployment\"=\"customer-infra-cluster-candidate-4-20-5pvqrv-67ffr4\"\n  \"ts\"=\"2026-06-12 13:49:04.475079\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-148\" \"deployment\"=\"msi-deployment-148\"\n  \"ts\"=\"2026-06-12 13:49:04.599107\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-06-12 13:49:05.015191\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-06-12 13:49:05.015430\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-candidate-4-20-vpbqs7\"\n  \"ts\"=\"2026-06-12 14:02:43.842163\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"rg-candidate-4-20-vpbqs7\"\n  \"ts\"=\"2026-06-12 14:02:43.842189\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"rg-candidate-4-20-vpbqs7\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n{0xc00007a094, 0x2a}, 0x0, {0xc0007868e0, ...})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:496\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).RunSpec(_, {_, _}, {_, _, _}, {_, _}, {0xc00007a094, 0x2a}, ...)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite_patch.go:68\n            github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite.func1.1({0x2e5aa85?, 0xc000ab0d80?})\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260422120742-50dec0b5ecf3/pkg/ginkgo/util.go:84\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.runSpec({0x91e1708, 0xc000e157c0}, 0xc000a57380, 0x1)\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260422120742-50dec0b5ecf3/pkg/extension/extensiontests/spec.go:596\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260422120742-50dec0b5ecf3/pkg/extension/extensiontests/spec.go:237\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260422120742-50dec0b5ecf3/pkg/extension/extensiontests/spec.go:230\n\n          goroutine 88 [select]\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2(0xc000a57380?)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:131\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts in goroutine 87\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:128\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:137 @ 06/12/26 13:49:03.006\n  \"ts\"=\"2026-06-12 13:49:03.007583\" \"level\"=0 \"msg\"=\"===== TEST CASE ENDED: FAILURE =====\"\n  \"ts\"=\"2026-06-12 13:49:03.007634\" \"level\"=0 \"msg\"=\"===== CLEANUP BEGAN =====\"\n  \"ts\"=\"2026-06-12 13:49:03.007761\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-06-12 13:49:03.007887\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-candidate-4-20-5pvqrv-67ffr4\" \"resourceGroup\"=\"rg-candidate-4-20-vpbqs7\"\n  \"ts\"=\"2026-06-12 13:49:03.075105\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-candidate-4-20-5pvqrv-67ffr4\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-06-12 13:49:03.778877\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-d4ee2620a775558818d65bdad60edec7ff8775b05730402bc0de88c0d1c3d86f.yaml.gz\"\n  \"ts\"=\"2026-06-12 13:49:03.779022\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-d4ee2620a775558818d65bdad60edec7ff8775b05730402bc0de88c0d1c3d86f.yaml.gz\"\n  \"ts\"=\"2026-06-12 13:49:03.779140\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-06-12 13:49:03.781368\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-candidate-4-20-vpbqs7\"\n  \"ts\"=\"2026-06-12 13:49:03.781396\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-148\"\n  \"ts\"=\"2026-06-12 13:49:04.152510\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-candidate-4-20-vpbqs7\" \"deployment\"=\"pooledNonMsiScopedAssignments-34bborbknryhu\"\n  \"ts\"=\"2026-06-12 13:49:04.166599\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-148\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-06-12 13:49:04.243967\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-candidate-4-20-vpbqs7\" \"deployment\"=\"customer-infra-cluster-candidate-4-20-5pvqrv-67ffr4\"\n  \"ts\"=\"2026-06-12 13:49:04.475079\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-148\" \"deployment\"=\"msi-deployment-148\"\n  \"ts\"=\"2026-06-12 13:49:04.599107\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-06-12 13:49:05.015191\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-06-12 13:49:05.015430\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-candidate-4-20-vpbqs7\"\n  \"ts\"=\"2026-06-12 14:02:43.842163\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"rg-candidate-4-20-vpbqs7\"\n  \"ts\"=\"2026-06-12 14:02:43.842189\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"rg-candidate-4-20-vpbqs7\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","000aa5188, {0x914ca80, 0xc000125120}, 0xc001904a18?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc0019049e8 sp=0xc0019049b0 pc=0x319c0fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc000aa5188, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc001904c60 sp=0xc0019049e8 pc=0x319917b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc000aa5188, {0xc0006af000, 0x1000, 0xc000308ba0?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc001904cc8 sp=0xc001904c60 pc=0x319fa85\nnet/http.(*persistConn).Read(0xc000aff320, {0xc0006af000?, 0x3289b05?, 0x9145b00?})\n\t/usr/lib/golang/src/net/http/transport.go:2125 +0x47 fp=0xc001904d28 sp=0xc001904cc8 pc=0x3291407\nbufio.(*Reader).fill(0xc000f00480)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0xc001904d60 sp=0xc001904d28 pc=0x3031943\nbufio.(*Reader).Peek(0xc000f00480, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0xc001904d80 sp=0xc001904d60 pc=0x3031ad3\nnet/http.(*persistConn).readLoop(0xc000aff320)\n\t/usr/lib/golang/src/net/http/transport.go:2278 +0x172 fp=0xc001904fc8 sp=0xc001904d80 pc=0x3291f12\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x25 fp=0xc001904fe0 sp=0xc001904fc8 pc=0x32908a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc001904fe8 sp=0xc001904fe0 pc=0x2ed3cc1\ncreated by net/http.(*Transport).dialConn in goroutine 129\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x174f\n\ngoroutine 203 gp=0xc001600700 m=nil [sync.WaitGroup.Wait, 4 minutes]:\nruntime.gopark(0xc0015fb5b0?, 0x2e62874?, 0x80?, 0x24?, 0x200c0013e0bd0?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc0010f9550 sp=0xc0010f9530 pc=0x2ecb50e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:466\nruntime.semacquire1(0xc0013ea778, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x21d fp=0xc0010f95b8 sp=0xc0010f9550 pc=0x2ea759d\nsync.runtime_SemacquireWaitGroup(0xc000519e20?, 0x0?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0xc0010f95f0 sp=0xc0010f95b8 pc=0x2eccc2e\nsync.(*WaitGroup).Wait(0xc0013ea770)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0xc0010f9618 sp=0xc0010f95f0 pc=0x2ef0405\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups(0xc000b99180, {0x7fa434753b08, 0xc00159b860}, {{0xc000ab4fb0, 0x1, 0x1}, 0x34630b8a000, {0x861de1c, 0x8}, {{0x0, ...}, ...}})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:371 +0x252 fp=0xc0010f9730 sp=0xc0010f9618 pc=0x6bae3d2\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources(0xc000b99180, {0x7fa434753b08, 0xc00159b860})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262 +0x250 fp=0xc0010f9888 sp=0xc0010f9730 pc=0x6badb30\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources-fm({0x7fa434753b08?, 0xc00159b860?})\n\t\u003cautogenerated\u003e:1 +0x33 fp=0xc0010f98b0 sp=0xc0010f9888 pc=0x6bc9cd3\nruntime.call16(0xc00177cc90, 0xc00069bf70, 0x0, 0x0, 0x0, 0x10, 0xc0010f9b08)\n\t/usr/lib/golang/src/runtime/asm_amd64.s:774 +0x43 fp=0xc0010f98d0 sp=0xc0010f98b0 pc=0x2ed2223\nruntime.reflectcall(0x7ff4600?, 0xc00159b860?, 0x2?, 0x865590b?, 0x0?, 0x12?, 0x7bbaac0?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc0010f9910 sp=0xc0010f98d0 pc=0x2edaab6\nreflect.Value.call({0x746fa60?, 0xc00069bf70?, 0xc000e8f3c8?}, {0x86117f8, 0x4}, {0xc00190ff60, 0x1, 0xc000d1ed70?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6 fp=0xc0010f9e70 sp=0xc0010f9910 pc=0x2f0dc66\nreflect.Value.Call({0x746fa60?, 0xc00069bf70?, 0x86102f5?}, {0xc00190ff60?, 0xc000e8f3c8?, 0x0?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9 fp=0xc0010f9ee8 sp=0xc0010f9e70 pc=0x2f0cd59\ngithub.com/onsi/ginkgo/v2/internal.NewCleanupNode.func2({0x91fba30?, 0xc00159b860?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:589 +0x150 fp=0xc0010f9f88 sp=0xc0010f9ee8 pc=0x3331170\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc0010f9fe0 sp=0xc0010f9f88 pc=0x3343c3b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0010f9fe8 sp=0xc0010f9fe0 pc=0x2ed3cc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 47\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nrax    0xfffffffffffffffc\nrbx    0x0\nrcx    0x2ed5ae3\nrdx    0x0\nrdi    0xc7f2cb0\nrsi    0x80\nrbp    0x7fa43cb09cb8\nrsp    0x7fa43cb09c70\nr8     0x0\nr9     0x0\nr10    0x7fa43cb09ca8\nr11    0x202\nr12    0x7fa43cb09ca8\nr13    0x1\nr14    0xc000002c40\nr15    0x0\nrip    0x2ed5ae3\nrflags 0x202\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\n000aa5188, {0x914ca80, 0xc000125120}, 0xc001904a18?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc0019049e8 sp=0xc0019049b0 pc=0x319c0fe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc000aa5188, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc001904c60 sp=0xc0019049e8 pc=0x319917b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc000aa5188, {0xc0006af000, 0x1000, 0xc000308ba0?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc001904cc8 sp=0xc001904c60 pc=0x319fa85\nnet/http.(*persistConn).Read(0xc000aff320, {0xc0006af000?, 0x3289b05?, 0x9145b00?})\n\t/usr/lib/golang/src/net/http/transport.go:2125 +0x47 fp=0xc001904d28 sp=0xc001904cc8 pc=0x3291407\nbufio.(*Reader).fill(0xc000f00480)\n\t/usr/lib/golang/src/bufio/bufio.go:113 +0x103 fp=0xc001904d60 sp=0xc001904d28 pc=0x3031943\nbufio.(*Reader).Peek(0xc000f00480, 0x1)\n\t/usr/lib/golang/src/bufio/bufio.go:152 +0x53 fp=0xc001904d80 sp=0xc001904d60 pc=0x3031ad3\nnet/http.(*persistConn).readLoop(0xc000aff320)\n\t/usr/lib/golang/src/net/http/transport.go:2278 +0x172 fp=0xc001904fc8 sp=0xc001904d80 pc=0x3291f12\nnet/http.(*Transport).dialConn.gowrap2()\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x25 fp=0xc001904fe0 sp=0xc001904fc8 pc=0x32908a5\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc001904fe8 sp=0xc001904fe0 pc=0x2ed3cc1\ncreated by net/http.(*Transport).dialConn in goroutine 129\n\t/usr/lib/golang/src/net/http/transport.go:1947 +0x174f\n\ngoroutine 203 gp=0xc001600700 m=nil [sync.WaitGroup.Wait, 4 minutes]:\nruntime.gopark(0xc0015fb5b0?, 0x2e62874?, 0x80?, 0x24?, 0x200c0013e0bd0?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc0010f9550 sp=0xc0010f9530 pc=0x2ecb50e\nruntime.goparkunlock(...)\n\t/usr/lib/golang/src/runtime/proc.go:466\nruntime.semacquire1(0xc0013ea778, 0x0, 0x1, 0x0, 0x19)\n\t/usr/lib/golang/src/runtime/sema.go:192 +0x21d fp=0xc0010f95b8 sp=0xc0010f9550 pc=0x2ea759d\nsync.runtime_SemacquireWaitGroup(0xc000519e20?, 0x0?)\n\t/usr/lib/golang/src/runtime/sema.go:114 +0x2e fp=0xc0010f95f0 sp=0xc0010f95b8 pc=0x2eccc2e\nsync.(*WaitGroup).Wait(0xc0013ea770)\n\t/usr/lib/golang/src/sync/waitgroup.go:206 +0x85 fp=0xc0010f9618 sp=0xc0010f95f0 pc=0x2ef0405\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CleanupResourceGroups(0xc000b99180, {0x7fa434753b08, 0xc00159b860}, {{0xc000ab4fb0, 0x1, 0x1}, 0x34630b8a000, {0x861de1c, 0x8}, {{0x0, ...}, ...}})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:371 +0x252 fp=0xc0010f9730 sp=0xc0010f9618 pc=0x6bae3d2\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources(0xc000b99180, {0x7fa434753b08, 0xc00159b860})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262 +0x250 fp=0xc0010f9888 sp=0xc0010f9730 pc=0x6badb30\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).deleteCreatedResources-fm({0x7fa434753b08?, 0xc00159b860?})\n\t\u003cautogenerated\u003e:1 +0x33 fp=0xc0010f98b0 sp=0xc0010f9888 pc=0x6bc9cd3\nruntime.call16(0xc00177cc90, 0xc00069bf70, 0x0, 0x0, 0x0, 0x10, 0xc0010f9b08)\n\t/usr/lib/golang/src/runtime/asm_amd64.s:774 +0x43 fp=0xc0010f98d0 sp=0xc0010f98b0 pc=0x2ed2223\nruntime.reflectcall(0x7ff4600?, 0xc00159b860?, 0x2?, 0x865590b?, 0x0?, 0x12?, 0x7bbaac0?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc0010f9910 sp=0xc0010f98d0 pc=0x2edaab6\nreflect.Value.call({0x746fa60?, 0xc00069bf70?, 0xc000e8f3c8?}, {0x86117f8, 0x4}, {0xc00190ff60, 0x1, 0xc000d1ed70?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6 fp=0xc0010f9e70 sp=0xc0010f9910 pc=0x2f0dc66\nreflect.Value.Call({0x746fa60?, 0xc00069bf70?, 0x86102f5?}, {0xc00190ff60?, 0xc000e8f3c8?, 0x0?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9 fp=0xc0010f9ee8 sp=0xc0010f9e70 pc=0x2f0cd59\ngithub.com/onsi/ginkgo/v2/internal.NewCleanupNode.func2({0x91fba30?, 0xc00159b860?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:589 +0x150 fp=0xc0010f9f88 sp=0xc0010f9ee8 pc=0x3331170\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc0010f9fe0 sp=0xc0010f9f88 pc=0x3343c3b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0010f9fe8 sp=0xc0010f9fe0 pc=0x2ed3cc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 47\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nrax    0xfffffffffffffffc\nrbx    0x0\nrcx    0x2ed5ae3\nrdx    0x0\nrdi    0xc7f2cb0\nrsi    0x80\nrbp    0x7fa43cb09cb8\nrsp    0x7fa43cb09c70\nr8     0x0\nr9     0x0\nr10    0x7fa43cb09ca8\nr11    0x202\nr12    0x7fa43cb09ca8\nr13    0x1\nr14    0xc000002c40\nr15    0x0\nrip    0x2ed5ae3\nrflags 0x202\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"2927e1230f4c23845b3a115b81dcf9a431e8ba103a757b7cf502028052c367ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"0436315dc74fa6c0436fdaa96d62f874f6aa63ac3c53854d13ceac33d2eeceea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"86fef99216c8f4c9009e4e3ec1412f56411c451077079e66942fba541140e103","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"0229965880d4859fafdf34c4b45eca9f3e33ce9ca472554f5ac79c449c7ebf38","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6f962bc17a3105c00a9b0bd7f4b2aca4444ece86f9a14faacfeee24cd27ebd87","failure_pattern":"failed to wait for first cluster \"\u003ccluster\u003e\" to complete creation","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":6.818181818181818,"anchor_occurrences":3,"after_last_push_count":5,"trend_counts":[0,1,0,0,0,2,0,0],"trend_range":"2026-06-07..2026-06-14 UTC","prior_windows_present":2,"prior_window_starts":["2026-05-25","2026-06-01"],"prior_runs_affected":5,"prior_last_seen_at":"2026-06-03T02:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065511780783755264","occurred_at":"2026-06-12T19:09:18Z","signature_id":"3e3a3d4f62c5e8a64ff8b5cb02b93498aaf8a6753c80e2b6b8b3864addabc2a3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065388059075022848","occurred_at":"2026-06-12T10:57:20Z","signature_id":"3e3a3d4f62c5e8a64ff8b5cb02b93498aaf8a6753c80e2b6b8b3864addabc2a3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064061230653378560","occurred_at":"2026-06-08T19:05:20Z","signature_id":"3e3a3d4f62c5e8a64ff8b5cb02b93498aaf8a6753c80e2b6b8b3864addabc2a3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"53fa6318769bb2a786556b3fac950c44b85ab8ba4f6fa60b2949a759bcc916c1","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.19.33\": cannot downgrade from version 4.21.18 to 4.19.33; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":4.545454545454546,"anchor_occurrences":2,"after_last_push_count":5,"trend_counts":[1,1,1,0,0,0,0,0],"trend_range":"2026-06-07..2026-06-14 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-01"],"prior_runs_affected":5,"prior_last_seen_at":"2026-06-07T02:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0002b8420\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-bzsg8d\" resourcegroup=\"rg-np-dg-minor-bzsg8d-gwv8l6\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-bzsg8d-gwv8l6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-bzsg8d/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": invalid node pool version 4.19.33: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": cannot downgrade from version 4.21.18 to 4.19.33\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0002b8420\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-bzsg8d\" resourcegroup=\"rg-np-dg-minor-bzsg8d-gwv8l6\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-bzsg8d-gwv8l6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-bzsg8d/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": invalid node pool version 4.19.33: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": cannot downgrade from version 4.21.18 to 4.19.33\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00039e480\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-mkwzwg\" resourcegroup=\"rg-np-dg-minor-mkwzwg-pcf6kp\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-mkwzwg-pcf6kp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-mkwzwg/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": invalid node pool version 4.19.33: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": cannot downgrade from version 4.21.18 to 4.19.33\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00039e480\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-mkwzwg\" resourcegroup=\"rg-np-dg-minor-mkwzwg-pcf6kp\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-mkwzwg-pcf6kp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-mkwzwg/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": invalid node pool version 4.19.33: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": cannot downgrade from version 4.21.18 to 4.19.33\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","occurred_at":"2026-06-09T02:00:03Z","signature_id":"b51928eb0bc2acc1b2df6c1f7efd4a6f5eba2e47b73c6478484af03364201334","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2063803203840905216","occurred_at":"2026-06-08T02:00:02Z","signature_id":"ff9c5b843f68b5e12d95ab85b0d2bf1a7da55153e66799fcd28883700873be12","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"80f757ab340f6d75766a4d77caaaddd21d63ed517eb7806a91d6c90f11d6a624","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.19.33\": cannot downgrade from version 4.21.19 to 4.19.33; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":4.545454545454546,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,1,1,0,0,0],"trend_range":"2026-06-07..2026-06-14 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f04b20\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-c9l7d4\" resourcegroup=\"rg-np-dg-minor-c9l7d4-jlrlpp\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-c9l7d4-jlrlpp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-c9l7d4/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": invalid node pool version 4.19.33: cannot downgrade from current version 4.21.19\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": cannot downgrade from version 4.21.19 to 4.19.33\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f04b20\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-c9l7d4\" resourcegroup=\"rg-np-dg-minor-c9l7d4-jlrlpp\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-c9l7d4-jlrlpp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-c9l7d4/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": invalid node pool version 4.19.33: cannot downgrade from current version 4.21.19\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": cannot downgrade from version 4.21.19 to 4.19.33\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d0e180\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-hf2g5m\" resourcegroup=\"rg-np-dg-minor-hf2g5m-8ws4v8\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-hf2g5m-8ws4v8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-hf2g5m/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": invalid node pool version 4.19.33: cannot downgrade from current version 4.21.19\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": cannot downgrade from version 4.21.19 to 4.19.33\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d0e180\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-hf2g5m\" resourcegroup=\"rg-np-dg-minor-hf2g5m-8ws4v8\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-hf2g5m-8ws4v8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-hf2g5m/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": invalid node pool version 4.19.33: cannot downgrade from current version 4.21.19\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": cannot downgrade from version 4.21.19 to 4.19.33\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064890364723990528","occurred_at":"2026-06-11T02:00:01Z","signature_id":"d53636f590081bb00e9fd2ac616257940cd863f596f189021d78c5f0b277e8f4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064527988531138560","occurred_at":"2026-06-10T02:00:04Z","signature_id":"746774ea21644ed376c73a98681bb6f039f642ffbe49f99ada4286dde678302f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d81a996cdf791decc51216fb7be72b9db7c9b98c36c458e851bda1b4a4b600a6","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.21.17\": cannot downgrade from version 4.21.18 to 4.21.17; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":4.545454545454546,"anchor_occurrences":2,"after_last_push_count":7,"trend_counts":[1,1,1,0,0,0,0,0],"trend_range":"2026-06-07..2026-06-14 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-01"],"prior_runs_affected":7,"prior_last_seen_at":"2026-06-07T02:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a18200\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-9hkgrv\" resourcegroup=\"rg-np-downgrade-9hkgrv-4rg65k\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-9hkgrv-4rg65k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-9hkgrv/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": invalid node pool version 4.21.17: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": cannot downgrade from version 4.21.18 to 4.21.17\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a18200\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-9hkgrv\" resourcegroup=\"rg-np-downgrade-9hkgrv-4rg65k\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-9hkgrv-4rg65k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-9hkgrv/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": invalid node pool version 4.21.17: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": cannot downgrade from version 4.21.18 to 4.21.17\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000134780\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-z52pvc\" resourcegroup=\"rg-np-downgrade-z52pvc-w5x8s7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-z52pvc-w5x8s7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-z52pvc/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": invalid node pool version 4.21.17: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": cannot downgrade from version 4.21.18 to 4.21.17\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000134780\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-z52pvc\" resourcegroup=\"rg-np-downgrade-z52pvc-w5x8s7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-z52pvc-w5x8s7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-z52pvc/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": invalid node pool version 4.21.17: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": cannot downgrade from version 4.21.18 to 4.21.17\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","occurred_at":"2026-06-09T02:00:03Z","signature_id":"4d175bb2e3a50c064fd19145dad25d9bc38d60a209a424b50532bf368e56fea0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2063803203840905216","occurred_at":"2026-06-08T02:00:02Z","signature_id":"4e20a613518a8ebe401b69555164476e8a73a8384289308330ccf2025c327983","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"b86f9a264db85a731f5c6fdee94b159cf165b041d6a81b701f673ae62312bfa8","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.21.18\": cannot downgrade from version 4.21.19 to 4.21.18; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":4.545454545454546,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,1,1,0,0,0],"trend_range":"2026-06-07..2026-06-14 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.18\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00099a220\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-sh8sbm\" resourcegroup=\"rg-np-downgrade-sh8sbm-8xc2g7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-sh8sbm-8xc2g7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-sh8sbm/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.18\\\": invalid node pool version 4.21.18: cannot downgrade from current version 4.21.19\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.18\\\": cannot downgrade from version 4.21.19 to 4.21.18\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.18\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00099a220\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-sh8sbm\" resourcegroup=\"rg-np-downgrade-sh8sbm-8xc2g7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-sh8sbm-8xc2g7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-sh8sbm/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.18\\\": invalid node pool version 4.21.18: cannot downgrade from current version 4.21.19\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.18\\\": cannot downgrade from version 4.21.19 to 4.21.18\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.18\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dc8ea0\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-62m6dt\" resourcegroup=\"rg-np-downgrade-62m6dt-z875wp\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-62m6dt-z875wp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-62m6dt/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.18\\\": invalid node pool version 4.21.18: cannot downgrade from current version 4.21.19\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.18\\\": cannot downgrade from version 4.21.19 to 4.21.18\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.18\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000dc8ea0\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-62m6dt\" resourcegroup=\"rg-np-downgrade-62m6dt-z875wp\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-62m6dt-z875wp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-62m6dt/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.18\\\": invalid node pool version 4.21.18: cannot downgrade from current version 4.21.19\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.18\\\": cannot downgrade from version 4.21.19 to 4.21.18\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064890364723990528","occurred_at":"2026-06-11T02:00:01Z","signature_id":"b13fe9b95b3c5336cfc484bd9bfc10633a6825001520493a6e3175e5cb7725f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064527988531138560","occurred_at":"2026-06-10T02:00:04Z","signature_id":"fde6f29812e4e7abde0fa8fa5483cb40c1e00e1bb8affea6418d034b87fb7d83","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":4.545454545454546,"anchor_occurrences":2,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,2,0],"trend_range":"2026-06-07..2026-06-14 UTC","prior_windows_present":1,"prior_window_starts":["2026-05-18"],"prior_runs_affected":4,"prior_last_seen_at":"2026-05-21T20:06:27Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000df03f0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-bn486z, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-bn486z\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000df03f0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-bn486z, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-bn486z\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00181a420\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-vdrfcr, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-vdrfcr\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00181a420\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-vdrfcr, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-vdrfcr\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065932263165530112","occurred_at":"2026-06-13T23:00:09Z","signature_id":"f173a978707f057f0927ba957cfb5fa902ce5c16fe0653d9f33e5a374fdb405d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065615170868088832","occurred_at":"2026-06-13T02:00:08Z","signature_id":"63c9271b9e105dce0242df2a05a76a9e669a791aac70d4e5117c2fd4b66e1497","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":4.545454545454546,"anchor_occurrences":2,"after_last_push_count":5,"trend_counts":[1,1,1,0,0,0,0,0],"trend_range":"2026-06-07..2026-06-14 UTC","prior_windows_present":3,"prior_window_starts":["2026-05-11","2026-05-18","2026-06-01"],"prior_runs_affected":8,"prior_last_seen_at":"2026-06-07T02:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:205]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-qrwxz9\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000997620\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-92ldj.apps.aro.r8i7j7z8f3c5v9h.00gq.eastus2euap.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:205]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-qrwxz9\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000997620\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-92ldj.apps.aro.r8i7j7z8f3c5v9h.00gq.eastus2euap.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:205]: failed to verify simple web app runs on cluster \"cluster-nightly-5-0-49vmww\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0000d5080\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-h7tkh.apps.aro.g5x8x9n2r7b0d4z.mce0.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:205]: failed to verify simple web app runs on cluster \"cluster-nightly-5-0-49vmww\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0000d5080\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-h7tkh.apps.aro.g5x8x9n2r7b0d4z.mce0.uksouth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064310885882531840","occurred_at":"2026-06-09T11:37:22Z","signature_id":"fb044cc30a88a2114dc710251c14a97955fa8c99dd37770d29a352a18e9abce8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2064120297979645952","occurred_at":"2026-06-08T23:00:03Z","signature_id":"970d1553d83adb9b2c3ff9515db6e1558cd0b193b0df5b377bcf4e22b01a7423","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4a64d19b350de8c067fb8a8e314eb2a7fc1e0b167107254f97c7d0949bcf19fa","failure_pattern":"\u003c/html\u003e","search_query":"failure","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.272727272727273,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,2,0,0],"trend_range":"2026-06-07..2026-06-14 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:75]: failed to get latest install version for nightly channel: after 4 attempts: query nightly tags for 5.0.0-0.nightly-multi returned 503 Service Unavailable: \u003chtml\u003e\r\n  \u003chead\u003e\r\n    \u003cmeta name=\"viewport\" content=\"width=device-width, initial-scale=1\"\u003e\r\n\r\n    \u003cstyle type=\"text/css\"\u003e\r\n      body {\r\n        font-family: \"Helvetica Neue\", Helvetica, Arial, sans-serif;\r\n        line-height: 1.66666667;\r\n        font-size: 16px;\r\n        color: #333;\r\n        background-color: #fff;\r\n        margin: 2em 1em;\r\n      }\r\n      h1 {\r\n        font-size: 28px;\r\n        font-weight: 400;\r\n      }\r\n      p {\r\n        margin: 0 0 10px;\r\n      }\r\n      .alert.alert-info {\r\n        background-color: #F0F0F0;\r\n        margin-top: 30px;\r\n        padding: 30px;\r\n      }\r\n      .alert p {\r\n        padding-left: 35px;\r\n      }\r\n      ul {\r\n        padding-left: 51px;\r\n        position: relative;\r\n      }\r\n      li {\r\n        font-size: 14px;\r\n        margin-bottom: 1em;\r\n      }\r\n      p.info {\r\n        position: relative;\r\n        font-size: 20px;\r\n      }\r\n      p.info:before, p.info:after {\r\n        content: \"\";\r\n        left: 0;\r\n        position: absolute;\r\n        top: 0;\r\n      }\r\n      p.info:before {\r\n        background: #0066CC;\r\n        border-radius: 16px;\r\n        color: #fff;\r\n        content: \"i\";\r\n        font: bold 16px/24px serif;\r\n        height: 24px;\r\n        left: 0px;\r\n        text-align: center;\r\n        top: 4px;\r\n        width: 24px;\r\n      }\r\n\r\n      @media (min-width: 768px) {\r\n        body {\r\n          margin: 6em;\r\n        }\r\n      }\r\n    \u003c/style\u003e\r\n  \u003c/head\u003e\r\n  \u003cbody\u003e\r\n    \u003cdiv\u003e\r\n      \u003ch1\u003eApplication is not available\u003c/h1\u003e\r\n      \u003cp\u003eThe application is currently not serving requests at this endpoint. It may not have been started or is still starting.\u003c/p\u003e\r\n\r\n      \u003cdiv class=\"alert alert-info\"\u003e\r\n        \u003cp class=\"info\"\u003e\r\n          Possible reasons you are seeing this page:\r\n        \u003c/p\u003e\r\n        \u003cul\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eThe host doesn't exist.\u003c/strong\u003e\r\n            Make sure the hostname was typed correctly and that a route matching this hostname exists.\r\n          \u003c/li\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eThe host exists, but doesn't have a matching path.\u003c/strong\u003e\r\n            Check if the URL path was typed correctly and that the route was created using the desired path.\r\n          \u003c/li\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eRoute and path matches, but all pods are down.\u003c/strong\u003e\r\n            Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.\r\n          \u003c/li\u003e\r\n        \u003c/ul\u003e\r\n      \u003c/div\u003e\r\n    \u003c/div\u003e\r\n  \u003c/body\u003e\r\n\u003c/html\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:75]: failed to get latest install version for nightly channel: after 4 attempts: query nightly tags for 5.0.0-0.nightly-multi returned 503 Service Unavailable: \u003chtml\u003e\r\n  \u003chead\u003e\r\n    \u003cmeta name=\"viewport\" content=\"width=device-width, initial-scale=1\"\u003e\r\n\r\n    \u003cstyle type=\"text/css\"\u003e\r\n      body {\r\n        font-family: \"Helvetica Neue\", Helvetica, Arial, sans-serif;\r\n        line-height: 1.66666667;\r\n        font-size: 16px;\r\n        color: #333;\r\n        background-color: #fff;\r\n        margin: 2em 1em;\r\n      }\r\n      h1 {\r\n        font-size: 28px;\r\n        font-weight: 400;\r\n      }\r\n      p {\r\n        margin: 0 0 10px;\r\n      }\r\n      .alert.alert-info {\r\n        background-color: #F0F0F0;\r\n        margin-top: 30px;\r\n        padding: 30px;\r\n      }\r\n      .alert p {\r\n        padding-left: 35px;\r\n      }\r\n      ul {\r\n        padding-left: 51px;\r\n        position: relative;\r\n      }\r\n      li {\r\n        font-size: 14px;\r\n        margin-bottom: 1em;\r\n      }\r\n      p.info {\r\n        position: relative;\r\n        font-size: 20px;\r\n      }\r\n      p.info:before, p.info:after {\r\n        content: \"\";\r\n        left: 0;\r\n        position: absolute;\r\n        top: 0;\r\n      }\r\n      p.info:before {\r\n        background: #0066CC;\r\n        border-radius: 16px;\r\n        color: #fff;\r\n        content: \"i\";\r\n        font: bold 16px/24px serif;\r\n        height: 24px;\r\n        left: 0px;\r\n        text-align: center;\r\n        top: 4px;\r\n        width: 24px;\r\n      }\r\n\r\n      @media (min-width: 768px) {\r\n        body {\r\n          margin: 6em;\r\n        }\r\n      }\r\n    \u003c/style\u003e\r\n  \u003c/head\u003e\r\n  \u003cbody\u003e\r\n    \u003cdiv\u003e\r\n      \u003ch1\u003eApplication is not available\u003c/h1\u003e\r\n      \u003cp\u003eThe application is currently not serving requests at this endpoint. It may not have been started or is still starting.\u003c/p\u003e\r\n\r\n      \u003cdiv class=\"alert alert-info\"\u003e\r\n        \u003cp class=\"info\"\u003e\r\n          Possible reasons you are seeing this page:\r\n        \u003c/p\u003e\r\n        \u003cul\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eThe host doesn't exist.\u003c/strong\u003e\r\n            Make sure the hostname was typed correctly and that a route matching this hostname exists.\r\n          \u003c/li\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eThe host exists, but doesn't have a matching path.\u003c/strong\u003e\r\n            Check if the URL path was typed correctly and that the route was created using the desired path.\r\n          \u003c/li\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eRoute and path matches, but all pods are down.\u003c/strong\u003e\r\n            Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.\r\n          \u003c/li\u003e\r\n        \u003c/ul\u003e\r\n      \u003c/div\u003e\r\n    \u003c/div\u003e\r\n  \u003c/body\u003e\r\n\u003c/html\u003e","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:75]: failed to get latest install version for nightly channel: after 4 attempts: query nightly tags for 4.23.0-0.nightly-multi returned 503 Service Unavailable: \u003chtml\u003e\r\n  \u003chead\u003e\r\n    \u003cmeta name=\"viewport\" content=\"width=device-width, initial-scale=1\"\u003e\r\n\r\n    \u003cstyle type=\"text/css\"\u003e\r\n      body {\r\n        font-family: \"Helvetica Neue\", Helvetica, Arial, sans-serif;\r\n        line-height: 1.66666667;\r\n        font-size: 16px;\r\n        color: #333;\r\n        background-color: #fff;\r\n        margin: 2em 1em;\r\n      }\r\n      h1 {\r\n        font-size: 28px;\r\n        font-weight: 400;\r\n      }\r\n      p {\r\n        margin: 0 0 10px;\r\n      }\r\n      .alert.alert-info {\r\n        background-color: #F0F0F0;\r\n        margin-top: 30px;\r\n        padding: 30px;\r\n      }\r\n      .alert p {\r\n        padding-left: 35px;\r\n      }\r\n      ul {\r\n        padding-left: 51px;\r\n        position: relative;\r\n      }\r\n      li {\r\n        font-size: 14px;\r\n        margin-bottom: 1em;\r\n      }\r\n      p.info {\r\n        position: relative;\r\n        font-size: 20px;\r\n      }\r\n      p.info:before, p.info:after {\r\n        content: \"\";\r\n        left: 0;\r\n        position: absolute;\r\n        top: 0;\r\n      }\r\n      p.info:before {\r\n        background: #0066CC;\r\n        border-radius: 16px;\r\n        color: #fff;\r\n        content: \"i\";\r\n        font: bold 16px/24px serif;\r\n        height: 24px;\r\n        left: 0px;\r\n        text-align: center;\r\n        top: 4px;\r\n        width: 24px;\r\n      }\r\n\r\n      @media (min-width: 768px) {\r\n        body {\r\n          margin: 6em;\r\n        }\r\n      }\r\n    \u003c/style\u003e\r\n  \u003c/head\u003e\r\n  \u003cbody\u003e\r\n    \u003cdiv\u003e\r\n      \u003ch1\u003eApplication is not available\u003c/h1\u003e\r\n      \u003cp\u003eThe application is currently not serving requests at this endpoint. It may not have been started or is still starting.\u003c/p\u003e\r\n\r\n      \u003cdiv class=\"alert alert-info\"\u003e\r\n        \u003cp class=\"info\"\u003e\r\n          Possible reasons you are seeing this page:\r\n        \u003c/p\u003e\r\n        \u003cul\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eThe host doesn't exist.\u003c/strong\u003e\r\n            Make sure the hostname was typed correctly and that a route matching this hostname exists.\r\n          \u003c/li\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eThe host exists, but doesn't have a matching path.\u003c/strong\u003e\r\n            Check if the URL path was typed correctly and that the route was created using the desired path.\r\n          \u003c/li\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eRoute and path matches, but all pods are down.\u003c/strong\u003e\r\n            Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.\r\n          \u003c/li\u003e\r\n        \u003c/ul\u003e\r\n      \u003c/div\u003e\r\n    \u003c/div\u003e\r\n  \u003c/body\u003e\r\n\u003c/html\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:75]: failed to get latest install version for nightly channel: after 4 attempts: query nightly tags for 4.23.0-0.nightly-multi returned 503 Service Unavailable: \u003chtml\u003e\r\n  \u003chead\u003e\r\n    \u003cmeta name=\"viewport\" content=\"width=device-width, initial-scale=1\"\u003e\r\n\r\n    \u003cstyle type=\"text/css\"\u003e\r\n      body {\r\n        font-family: \"Helvetica Neue\", Helvetica, Arial, sans-serif;\r\n        line-height: 1.66666667;\r\n        font-size: 16px;\r\n        color: #333;\r\n        background-color: #fff;\r\n        margin: 2em 1em;\r\n      }\r\n      h1 {\r\n        font-size: 28px;\r\n        font-weight: 400;\r\n      }\r\n      p {\r\n        margin: 0 0 10px;\r\n      }\r\n      .alert.alert-info {\r\n        background-color: #F0F0F0;\r\n        margin-top: 30px;\r\n        padding: 30px;\r\n      }\r\n      .alert p {\r\n        padding-left: 35px;\r\n      }\r\n      ul {\r\n        padding-left: 51px;\r\n        position: relative;\r\n      }\r\n      li {\r\n        font-size: 14px;\r\n        margin-bottom: 1em;\r\n      }\r\n      p.info {\r\n        position: relative;\r\n        font-size: 20px;\r\n      }\r\n      p.info:before, p.info:after {\r\n        content: \"\";\r\n        left: 0;\r\n        position: absolute;\r\n        top: 0;\r\n      }\r\n      p.info:before {\r\n        background: #0066CC;\r\n        border-radius: 16px;\r\n        color: #fff;\r\n        content: \"i\";\r\n        font: bold 16px/24px serif;\r\n        height: 24px;\r\n        left: 0px;\r\n        text-align: center;\r\n        top: 4px;\r\n        width: 24px;\r\n      }\r\n\r\n      @media (min-width: 768px) {\r\n        body {\r\n          margin: 6em;\r\n        }\r\n      }\r\n    \u003c/style\u003e\r\n  \u003c/head\u003e\r\n  \u003cbody\u003e\r\n    \u003cdiv\u003e\r\n      \u003ch1\u003eApplication is not available\u003c/h1\u003e\r\n      \u003cp\u003eThe application is currently not serving requests at this endpoint. It may not have been started or is still starting.\u003c/p\u003e\r\n\r\n      \u003cdiv class=\"alert alert-info\"\u003e\r\n        \u003cp class=\"info\"\u003e\r\n          Possible reasons you are seeing this page:\r\n        \u003c/p\u003e\r\n        \u003cul\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eThe host doesn't exist.\u003c/strong\u003e\r\n            Make sure the hostname was typed correctly and that a route matching this hostname exists.\r\n          \u003c/li\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eThe host exists, but doesn't have a matching path.\u003c/strong\u003e\r\n            Check if the URL path was typed correctly and that the route was created using the desired path.\r\n          \u003c/li\u003e\r\n          \u003cli\u003e\r\n            \u003cstrong\u003eRoute and path matches, but all pods are down.\u003c/strong\u003e\r\n            Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.\r\n          \u003c/li\u003e\r\n        \u003c/ul\u003e\r\n      \u003c/div\u003e\r\n    \u003c/div\u003e\r\n  \u003c/body\u003e\r\n\u003c/html\u003e"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065303808690360320","occurred_at":"2026-06-12T05:22:54Z","signature_id":"957e96f8e5ed07f64b0bc7ba6f4d11df6a74a38b64d1fe93779ad1721139ec68","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2065303808690360320","occurred_at":"2026-06-12T05:22:54Z","signature_id":"d581ddd3d399eeafcb2f33e8d53dd7bd9bc29e90ca98a6492e7bb1b8cb23026e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"baee198396fe46d09318662e813056f6a99076e40cebbc3f321bb246d7fd7f20","failure_pattern":"Post \"\u003curl\u003e\": dial tcp \u003cip\u003e:\u003cport\u003e: i/o timeout","search_query":"Post \"https://api.n1i3e7e9x8g0b6s.ci57.brazilsouth.aroapp-hcp.io:443/apis/authentication.k8s.io/v1/selfsubjectreviews\": dial tcp 74.163.139.192:443: i/o timeout","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.272727272727273,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-06-07..2026-06-14 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:236]: should be able to create SelfSubjectReview for admin credential 2\nUnexpected error:\n    \u003c*url.Error | 0xc000999560\u003e: \n    Post \"https://api.n1i3e7e9x8g0b6s.ci57.brazilsouth.aroapp-hcp.io:443/apis/authentication.k8s.io/v1/selfsubjectreviews\": dial tcp 74.163.139.192:443: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:236]: should be able to create SelfSubjectReview for admin credential 2\nUnexpected error:\n    \u003c*url.Error | 0xc000999560\u003e: \n    Post \"https://api.n1i3e7e9x8g0b6s.ci57.brazilsouth.aroapp-hcp.io:443/apis/authentication.k8s.io/v1/selfsubjectreviews\": dial tcp 74.163.139.192:443: i/o timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064995434417360896","occurred_at":"2026-06-11T08:57:31Z","signature_id":"5c44ef45a71fd7d703be1cefc7a23cb14c5954fb9c9fedad6bb00fd770768efe","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}