{"meta":{"start_date":"2026-07-02","end_date":"2026-07-02","start_at":"2026-07-02T00:00:00Z","end_at":"2026-07-03T00:00:00Z","timezone":"UTC","generated_at":"2026-08-28T01:04:25Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":3,"failed_runs":3,"raw_occurrences":36,"matched_occurrences":36,"runs_affected":3},"rows":[{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-5-0-x94n2m, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-x94n2m in resource group rg-candidate-5-0-srwxkf, error: failed waiting for cluster=\"cluster-candidate-5-0-x94n2m\" in resourcegroup=\"rg-candidate-5-0-srwxkf\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-x94n2m in resource group rg-candidate-5-0-srwxkf, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":23,"runs_affected":2,"failed_runs":2,"run_impact_percent":66.66666666666667,"anchor_occurrences":23,"after_last_push_count":23,"trend_counts":[0,0,0,0,0,0,23],"trend_range":"2026-06-26..2026-07-02 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-01","2026-06-08","2026-06-15","2026-06-22"],"prior_runs_affected":56,"prior_last_seen_at":"2026-06-28T20:35:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-4-23-mw7g87/cluster-nightly-4-23-b45mnx should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a30de0\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-b45mnx, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-b45mnx in resource group rg-nightly-4-23-mw7g87, error: failed waiting for cluster=\"cluster-nightly-4-23-b45mnx\" in resourcegroup=\"rg-nightly-4-23-mw7g87\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-b45mnx in resource group rg-nightly-4-23-mw7g87, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-4-23-mw7g87/cluster-nightly-4-23-b45mnx should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a30de0\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-b45mnx, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-b45mnx in resource group rg-nightly-4-23-mw7g87, error: failed waiting for cluster=\"cluster-nightly-4-23-b45mnx\" in resourcegroup=\"rg-nightly-4-23-mw7g87\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-b45mnx in resource group rg-nightly-4-23-mw7g87, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-5-0-mn6b27/cluster-nightly-5-0-7sf6qf should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001486090\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-7sf6qf, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-7sf6qf in resource group rg-nightly-5-0-mn6b27, error: failed waiting for cluster=\"cluster-nightly-5-0-7sf6qf\" in resourcegroup=\"rg-nightly-5-0-mn6b27\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-7sf6qf in resource group rg-nightly-5-0-mn6b27, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-5-0-mn6b27/cluster-nightly-5-0-7sf6qf should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001486090\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-7sf6qf, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-7sf6qf in resource group rg-nightly-5-0-mn6b27, error: failed waiting for cluster=\"cluster-nightly-5-0-7sf6qf\" in resourcegroup=\"rg-nightly-5-0-mn6b27\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-7sf6qf in resource group rg-nightly-5-0-mn6b27, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-srwxkf/cluster-candidate-5-0-x94n2m should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f8ec60\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-x94n2m, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-x94n2m in resource group rg-candidate-5-0-srwxkf, error: failed waiting for cluster=\"cluster-candidate-5-0-x94n2m\" in resourcegroup=\"rg-candidate-5-0-srwxkf\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-x94n2m in resource group rg-candidate-5-0-srwxkf, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-srwxkf/cluster-candidate-5-0-x94n2m should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f8ec60\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-x94n2m, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-x94n2m in resource group rg-candidate-5-0-srwxkf, error: failed waiting for cluster=\"cluster-candidate-5-0-x94n2m\" in resourcegroup=\"rg-candidate-5-0-srwxkf\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-x94n2m in resource group rg-candidate-5-0-srwxkf, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2072817623409627136","occurred_at":"2026-07-02T23:00:07Z","signature_id":"9c642aeecd491056fbfe705316809dbe51e9422fb7f0a5c6c6779e04e3ed9881","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2072817623409627136","occurred_at":"2026-07-02T23:00:07Z","signature_id":"dd1933f590b4703b4f8c0307d337cde0e6b10a3f990784327dfd08144b177cdb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"015e166a76737e30b961a38d806b0964b5cc3ad1cf79b8482bedf373e2030bbc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"06d92daa09c1eed853993a47f69843962bf6d66f9945c41bf48a6a548d5fe7b9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"14c21f15e9434a6910cb1114c7684b9d607b475ee9c14813e848685559d7878e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"1de2faf23b33bc6c87b36a77ebdca810161959ee1aaa6e3c020033138edeac5e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"226e6ffd6812a3ef28043a1a4c200c341f0ef4018961b9d8662c1a5da36ac113","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"255618ec4ebf7b0ecd0fb7a3a795baa2f0276403226cd6a68c97398af7b1139f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"44cb7199f7d23201a03780743da5a4b84d186ecaa9259366ae2c60fcee299f81","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"52f4d90a959f5cf2d2b0cc4b333870046c65c86c5a88f17fc1db801897ed0ea4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"545d3997a8ffcec4e3111f8a8849d3cd147665395cd047782c7dff431440df0a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"5a179b042691135876873d8ae138d8a775bbcf9443e5e8d36eb14aa74125a5b7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"64426cc5c6cae9c3bcee303173a756e9989c85da03d58ac95f6a00f2f5eab2ee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"6a47e979689cec901720c823c5a10b6cc83dde69f728d783bb50f6d40babe15d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"6bdfa777c0a374509ddca89a0a992051ed6ea9edd61aeff6860ab0ec8d14150d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"710eb07299f9e7fcccd0a60d262ce740a7670d6f1bda8ba26772d42ed6c83dbf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"7454cedf8d14dd8d0dfc6539705a44f3b74b89e52440fe70bd0b7cbdd456818a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"84ace348ec7dcf2578f5dbfc02f4ad8af4daea59abb97cded99dc421cd0969a4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"905d2e26de4c5cc3c2d15b7afb974928951f6434526787b6ca6d01d8964dde54","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"a42c1905561bc136ed5b43dcd171060fc0fe9a117e4318689fe39b07399cdd2a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"c2a4a6d84da3879b0f6b30229f86315c39297f05eea8a6881610352584073c14","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"e390fbd741484007dc07eba39f0e87349ff79e90ef6a2e49fb96edd11b52567a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"e675e116113892b91c6f868ae4a59fc1d683a48147b5d92ce9f325640a4c83be","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-9gq4xd, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-9gq4xd\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":7,"runs_affected":3,"failed_runs":3,"run_impact_percent":100,"anchor_occurrences":7,"after_last_push_count":7,"trend_counts":[0,0,0,0,0,0,7],"trend_range":"2026-06-26..2026-07-02 UTC","prior_windows_present":3,"prior_window_starts":["2026-06-08","2026-06-15","2026-06-22"],"prior_runs_affected":29,"prior_last_seen_at":"2026-06-28T23:00:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:136]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0026f1410\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-45ldvl, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-45ldvl\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:136]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0026f1410\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-45ldvl, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-45ldvl\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:148]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002a0b620\u003e: \n    failed to create NodePool autoscale-az, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-n9df2b, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-n9df2b\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:148]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002a0b620\u003e: \n    failed to create NodePool autoscale-az, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-n9df2b, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-n9df2b\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-8skd5b\" for cluster \"cluster-nightly-4-20-8skd5b\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0028a7680\u003e: \n    failed to create NodePool np-8skd5b, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-8skd5b in resource group rg-nightly-4-20-dvmp6t, error: failed waiting for nodepool=\"np-8skd5b\" for cluster \"cluster-nightly-4-20-8skd5b\" in resourcegroup=\"rg-nightly-4-20-dvmp6t\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-8skd5b\" for cluster \"cluster-nightly-4-20-8skd5b\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0028a7680\u003e: \n    failed to create NodePool np-8skd5b, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-8skd5b in resource group rg-nightly-4-20-dvmp6t, error: failed waiting for nodepool=\"np-8skd5b\" for cluster \"cluster-nightly-4-20-8skd5b\" in resourcegroup=\"rg-nightly-4-20-dvmp6t\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2072817623409627136","occurred_at":"2026-07-02T23:00:07Z","signature_id":"0ef93110a307d8b79085b84a5e3876b8e56f8d8038e750d09e6a3b0163defd02","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2072817623409627136","occurred_at":"2026-07-02T23:00:07Z","signature_id":"4f4e657b64ce77c2e5de4988ee6545355a8f547200320d9d1df6d8a810b75c05","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2072817623409627136","occurred_at":"2026-07-02T23:00:07Z","signature_id":"6628cfa66b0512150e846a4feec5ea64bc961167bdefe3a9a1163377faea7b35","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2072817623409627136","occurred_at":"2026-07-02T23:00:07Z","signature_id":"74fcb433d90a2b18960a730b85ce319e1b6b94bd27219d6b71855ef514732ead","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2072817623409627136","occurred_at":"2026-07-02T23:00:07Z","signature_id":"e1d04d9a0f63ce8129c23d0b2290940280de6c25b953658da6f638196a226345","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2072623469333319680","occurred_at":"2026-07-02T10:08:37Z","signature_id":"e794c6313b015bbf59f2a864560e86d9935145d6c9231b17bf2e9a22a9d126ed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"22038c452c7893cb3508907871c33f9c0c1e5ded9a5e4bd2002cda94e9dae1ca","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7f0e44476112ce4e675f687843407c03482d80d265239de5b2ddbd45c441a04e","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-m5x4n6\" to finish deleting: context canceled","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,0,5],"trend_range":"2026-06-26..2026-07-02 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-01","2026-06-08","2026-06-15","2026-06-22"],"prior_runs_affected":19,"prior_last_seen_at":"2026-06-28T23:00:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00190c168\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-19-5wmrnm\" in resourcegroup=\"rg-zstream-upgrade-4-19-l9cvg8\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00190c168\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-19-5wmrnm\" in resourcegroup=\"rg-zstream-upgrade-4-19-l9cvg8\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000ccc2e8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-sn4mb4\" in resourcegroup=\"rg-zstream-upgrade-4-20-4p2hrd\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000ccc2e8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-sn4mb4\" in resourcegroup=\"rg-zstream-upgrade-4-20-4p2hrd\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00158e198\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-m5x4n6\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00158e198\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-m5x4n6\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"0f036fe9d0e2ee8ff939234c026c4bdd0b264895f82429802d28f8a26f893b4d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"2a72bf1b75e36fd5fd1863c2b636eaed3a8c405dbf3f2dd0456b2886b609a8af","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"2e0cdbc687982a8782f9fe52389fd1adf21653c5b09dd4ae4c9d03cd5b5d3ca5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"9e84ee7850e2604f7ccd45b1147cc172fb4b42df018315232368eca4a1d96bb6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072500530168467456","occurred_at":"2026-07-02T02:00:06Z","signature_id":"d3bbf56b268c4ceb880458eda39d3aa497fcd60f11f147386b431eed8d9bb730","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"9d76b8574a881c48ca814c73a5f78cfd9c34fbf0c2d12d09c9f9b6e856adedcf","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-06-26..2026-07-02 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001e38480\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-zvscj7, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-zvscj7\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001e38480\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-zvscj7, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-zvscj7\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2072817623409627136","occurred_at":"2026-07-02T23:00:07Z","signature_id":"a9da48daa50b2da76bc04dbf40367321003cd48ea99940b476bf6bac8efee68d","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}