{"meta":{"start_date":"2026-06-29","end_date":"2026-06-29","start_at":"2026-06-29T00:00:00Z","end_at":"2026-06-30T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T23:05:02Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":3,"failed_runs":3,"raw_occurrences":22,"matched_occurrences":22,"runs_affected":3},"rows":[{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-vwr2vw, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-vwr2vw in resource group rg-candidate-4-20-5xbch6, error: failed waiting for cluster=\"cluster-candidate-4-20-vwr2vw\" in resourcegroup=\"rg-candidate-4-20-5xbch6\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-vwr2vw in resource group rg-candidate-4-20-5xbch6, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":12,"runs_affected":3,"failed_runs":3,"run_impact_percent":100,"anchor_occurrences":12,"after_last_push_count":12,"trend_counts":[0,0,0,0,0,0,12],"trend_range":"2026-06-23..2026-06-29 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-01","2026-06-08","2026-06-15","2026-06-22"],"prior_runs_affected":47,"prior_last_seen_at":"2026-06-28T20:35:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-4-22-c8xmw4/cluster-nightly-4-22-cw6fzn should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000900db0\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-cw6fzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-22-cw6fzn in resource group rg-nightly-4-22-c8xmw4, error: failed waiting for cluster=\"cluster-nightly-4-22-cw6fzn\" in resourcegroup=\"rg-nightly-4-22-c8xmw4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-22-cw6fzn in resource group rg-nightly-4-22-c8xmw4, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-4-22-c8xmw4/cluster-nightly-4-22-cw6fzn should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000900db0\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-cw6fzn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-22-cw6fzn in resource group rg-nightly-4-22-c8xmw4, error: failed waiting for cluster=\"cluster-nightly-4-22-cw6fzn\" in resourcegroup=\"rg-nightly-4-22-c8xmw4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-22-cw6fzn in resource group rg-nightly-4-22-c8xmw4, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000382210\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-shgf7d\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-shgf7d, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000382210\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-shgf7d\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-shgf7d, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f00c00\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-2k7tp4, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-2k7tp4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-2k7tp4, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f00c00\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-2k7tp4, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-2k7tp4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-2k7tp4, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2071730535067029504","occurred_at":"2026-06-29T23:00:25Z","signature_id":"317708af01bf7e5bb12d7dfde8e518f4bd19cb16aa9784d148488dadf1635f16","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2071730535067029504","occurred_at":"2026-06-29T23:00:25Z","signature_id":"92fd469e506c95d2c2aa6fc986edce7be479fd2bc150a302baebe0dead7e8b52","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2071730535067029504","occurred_at":"2026-06-29T23:00:25Z","signature_id":"a3cfa483b6e409ed034c431e624224ae36c84174fb3dd7342c38d8ce6dad567f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2071730535067029504","occurred_at":"2026-06-29T23:00:25Z","signature_id":"b75c7941a8b510cbd34461b71437b37fa0e583149ec6a5f2ecafbc7727c80497","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2071730535067029504","occurred_at":"2026-06-29T23:00:25Z","signature_id":"c345470b7f99b4836a55aed0c0a70865a7dfce624d03e58858c3e43ca1c239f7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2071469428922388480","occurred_at":"2026-06-29T05:42:52Z","signature_id":"6a75293eeffc81dd4b1aa5dd7a004b58c79e36c04a7c0a1c6db0fd55f7d3e44a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2071469428922388480","occurred_at":"2026-06-29T05:42:52Z","signature_id":"97000e185b82048fb8f964d1ca82c848b646e6a0814e742aa3917023c89fe8f6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2071469428922388480","occurred_at":"2026-06-29T05:42:52Z","signature_id":"f1776b85f213157978cee4a8835a72bf7bd7fc42e640670d17e91e12995cf4cc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2071413440525635584","occurred_at":"2026-06-29T02:00:23Z","signature_id":"07aa1f6a3d4cf97ef2d31967107729bfe69475813429a30820775411633552ab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2071413440525635584","occurred_at":"2026-06-29T02:00:23Z","signature_id":"3f996b1c729f5a788ed8e839fd5f025dd6b820c8b14b3f4c7b91e394600f7454","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2071413440525635584","occurred_at":"2026-06-29T02:00:23Z","signature_id":"4257a1d0b07bc90bbfcb4e9638ec06e439261a2aa967847ffa7c288a93a43521","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2071413440525635584","occurred_at":"2026-06-29T02:00:23Z","signature_id":"67b6f7160d05991c0015c3ab384280279a4ffb74d00fed4f472d23b7549378eb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7f0e44476112ce4e675f687843407c03482d80d265239de5b2ddbd45c441a04e","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-2k2kzp\" to finish deleting: context canceled","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":5,"runs_affected":2,"failed_runs":2,"run_impact_percent":66.66666666666667,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,0,5],"trend_range":"2026-06-23..2026-06-29 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-01","2026-06-08","2026-06-15","2026-06-22"],"prior_runs_affected":15,"prior_last_seen_at":"2026-06-28T23:00:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001528138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-cdcrrh\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001528138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-cdcrrh\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00157c330\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listdp8zz5\" in resourcegroup=\"ea-list-hmnqqz\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00157c330\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listdp8zz5\" in resourcegroup=\"ea-list-hmnqqz\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00184a3f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-2k2kzp\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00184a3f0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-2k2kzp\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2071469428922388480","occurred_at":"2026-06-29T05:42:52Z","signature_id":"02577a99a48fca7133f2f6928b20f5bf2320c784312caa1429833e5a385819a2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2071469428922388480","occurred_at":"2026-06-29T05:42:52Z","signature_id":"e1e9ae3095783fc86dfeedd64469efce6464f9f29d8465c840faf8d79b2f563f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2071469428922388480","occurred_at":"2026-06-29T05:42:52Z","signature_id":"f0c7124139d9375dc09bc700c0ed9abf34f4e98489a08e75dcb51e31b2f8beb5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2071413440525635584","occurred_at":"2026-06-29T02:00:23Z","signature_id":"364615ca63e3a4aabf53e7bca5026c2e8815f30048d8d233c6910a2c937fe053","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2071413440525635584","occurred_at":"2026-06-29T02:00:23Z","signature_id":"ca387e5647a360e992abe1287000224b212db51b3c739c28f9680a8152aedab9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-noaz in resource group np-autoscaling-lxrt2n, error: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-lxrt2n\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":66.66666666666667,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-06-23..2026-06-29 UTC","prior_windows_present":3,"prior_window_starts":["2026-06-08","2026-06-15","2026-06-22"],"prior_runs_affected":20,"prior_last_seen_at":"2026-06-28T23:00:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:151]: failed to create node pool npupgrade-4-20 with version 4.20.27\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001583e60\u003e: \n    failed to create NodePool npupgrade-4-20, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-20 in resource group rg-np-version-upgrade-bxxxvv-s5c7c7, error: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-bxxxvv\" in resourcegroup=\"rg-np-version-upgrade-bxxxvv-s5c7c7\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:151]: failed to create node pool npupgrade-4-20 with version 4.20.27\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001583e60\u003e: \n    failed to create NodePool npupgrade-4-20, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-20 in resource group rg-np-version-upgrade-bxxxvv-s5c7c7, error: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-bxxxvv\" in resourcegroup=\"rg-np-version-upgrade-bxxxvv-s5c7c7\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:214]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014e7d70\u003e: \n    failed to create NodePool autoscale-noaz, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-noaz in resource group np-autoscaling-lxrt2n, error: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-lxrt2n\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:214]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014e7d70\u003e: \n    failed to create NodePool autoscale-noaz, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-noaz in resource group np-autoscaling-lxrt2n, error: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-lxrt2n\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:259]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002692ae0\u003e: \n    failed to create NodePool oidc-wi-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool oidc-wi-np in resource group oidc-wi-hlnm8w, error: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-hlnm8w\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:259]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002692ae0\u003e: \n    failed to create NodePool oidc-wi-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool oidc-wi-np in resource group oidc-wi-hlnm8w, error: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-hlnm8w\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2071469428922388480","occurred_at":"2026-06-29T05:42:52Z","signature_id":"3004b662385dfba1049c02f252fcfb4434ff9cdfddce5bad08415274d64ba4bc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2071413440525635584","occurred_at":"2026-06-29T02:00:23Z","signature_id":"5ee9aa735f7cb8c2e6b8cceb774ceccf863fefe0ae013efb502f6f07f95fe45a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2071413440525635584","occurred_at":"2026-06-29T02:00:23Z","signature_id":"acf8fb68a61e87ce1c1409983e3085fb2937b7654f1785c6a90d9d387e68e90a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"d6ec87c7d972149dae5ab13de0394af8c0dc25683fd32e5ecca933f42f5cfc45","failure_pattern":"DeleteResourceGroup timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-zbvtqd, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-06-23..2026-06-29 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:145]: failed to delete customer resource group \"cx-rg-cluster-zbvtqd\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00256ebd0\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-zbvtqd, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:145]: failed to delete customer resource group \"cx-rg-cluster-zbvtqd\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00256ebd0\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-zbvtqd, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2071413440525635584","occurred_at":"2026-06-29T02:00:23Z","signature_id":"688fb9d825aee0557c071af784a5eaca516c6fd6915a16d770d552c1d85bf183","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"83447d3d19023b63c00022eba710d55e1a0090317aba9cb3e48328dc0b521261","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-bh2m2l\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-06-23..2026-06-29 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:140]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a8ad20\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-bh2m2l\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:140]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a8ad20\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-bh2m2l\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2071413440525635584","occurred_at":"2026-06-29T02:00:23Z","signature_id":"67c7ff953fa8d0d3d7c916636a8ef416983e47296bd8f68d66a130f3f9d272eb","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}