{"meta":{"start_date":"2026-07-01","end_date":"2026-07-01","start_at":"2026-07-01T00:00:00Z","end_at":"2026-07-02T00:00:00Z","timezone":"UTC","generated_at":"2026-08-27T00:46:17Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":3,"failed_runs":3,"raw_occurrences":11,"matched_occurrences":11,"runs_affected":3},"rows":[{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-wb42z8, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-wb42z8 in resource group rg-candidate-4-20-w2zlp6, error: failed waiting for cluster=\"cluster-candidate-4-20-wb42z8\" in resourcegroup=\"rg-candidate-4-20-w2zlp6\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-wb42z8 in resource group rg-candidate-4-20-w2zlp6, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":8,"runs_affected":3,"failed_runs":3,"run_impact_percent":100,"anchor_occurrences":8,"after_last_push_count":8,"trend_counts":[0,0,0,0,0,0,8],"trend_range":"2026-06-25..2026-07-01 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-01","2026-06-08","2026-06-15","2026-06-22"],"prior_runs_affected":54,"prior_last_seen_at":"2026-06-28T20:35:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:88]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015f4570\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-l8w8dj, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-l8w8dj\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-l8w8dj, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:88]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015f4570\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-l8w8dj, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-l8w8dj\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-l8w8dj, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00102a270\u003e: \n    failed to create HCP cluster negative-tests-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-hnckp7, error: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-hnckp7\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-hnckp7, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00102a270\u003e: \n    failed to create HCP cluster negative-tests-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-hnckp7, error: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-hnckp7\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-hnckp7, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-4-20-pdblgm/cluster-nightly-4-20-h74hnn should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015da7e0\u003e: \n    failed to create HCP cluster cluster-nightly-4-20-h74hnn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-20-h74hnn in resource group rg-nightly-4-20-pdblgm, error: failed waiting for cluster=\"cluster-nightly-4-20-h74hnn\" in resourcegroup=\"rg-nightly-4-20-pdblgm\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-20-h74hnn in resource group rg-nightly-4-20-pdblgm, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-4-20-pdblgm/cluster-nightly-4-20-h74hnn should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015da7e0\u003e: \n    failed to create HCP cluster cluster-nightly-4-20-h74hnn, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-20-h74hnn in resource group rg-nightly-4-20-pdblgm, error: failed waiting for cluster=\"cluster-nightly-4-20-h74hnn\" in resourcegroup=\"rg-nightly-4-20-pdblgm\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-20-h74hnn in resource group rg-nightly-4-20-pdblgm, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2072455231878205440","occurred_at":"2026-07-01T23:00:06Z","signature_id":"13f252fb26293a3656cff84fb2c33610892158a7b7f607781bbfa60b5521e142","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2072455231878205440","occurred_at":"2026-07-01T23:00:06Z","signature_id":"82685272f64d401cc1c30c9010c2a0f8b9fd7c44acea09e700558bdf4c005c52","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2072455231878205440","occurred_at":"2026-07-01T23:00:06Z","signature_id":"c806fd98239986789bf41d9eff508fed04a099f035e7570ab6c2143267180057","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2072268053562789888","occurred_at":"2026-07-01T10:36:19Z","signature_id":"179748c6eaa96fec88cb0a08f84accb68c59da754e210328646fbe50a0a8646d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2072268053562789888","occurred_at":"2026-07-01T10:36:19Z","signature_id":"2045c690cd234706bdcfec02ae7397f7171568c81b62368a6da5a2c9a21c9016","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2072268053562789888","occurred_at":"2026-07-01T10:36:19Z","signature_id":"f09f7b6a32c43e618233fb1a21100b38f4b4f34f8d11721481798334de9befe1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072138226172170240","occurred_at":"2026-07-01T02:00:26Z","signature_id":"2375c473c73799a3e320d5a34bcc55481c2348213bdced8af98dce8cd2dd270d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072138226172170240","occurred_at":"2026-07-01T02:00:26Z","signature_id":"b8ad8025a70641a9958a392779dc5187b8be351aa28ceb15948144d058e8ca80","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-5zzp9q, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5zzp9q\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":100,"anchor_occurrences":3,"after_last_push_count":29,"trend_counts":[0,2,2,2,3,3,3],"trend_range":"2026-06-25..2026-07-01 UTC","prior_windows_present":3,"prior_window_starts":["2026-06-08","2026-06-15","2026-06-22"],"prior_runs_affected":26,"prior_last_seen_at":"2026-06-28T23:00:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:148]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0021b3e90\u003e: \n    failed to create NodePool autoscale-az, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-wpj5tc, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-wpj5tc\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:148]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0021b3e90\u003e: \n    failed to create NodePool autoscale-az, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-wpj5tc, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-wpj5tc\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:148]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00281a3c0\u003e: \n    failed to create NodePool autoscale-az, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-5zzp9q, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5zzp9q\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:148]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00281a3c0\u003e: \n    failed to create NodePool autoscale-az, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-5zzp9q, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5zzp9q\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:148]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0026d1770\u003e: \n    failed to create NodePool autoscale-az, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-nxfxts, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-nxfxts\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:148]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0026d1770\u003e: \n    failed to create NodePool autoscale-az, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-nxfxts, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-nxfxts\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2072455231878205440","occurred_at":"2026-07-01T23:00:06Z","signature_id":"62cc4b0dca9b5293cfd1df95c9487385db100958ff11ceac4196604ec031218c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2072268053562789888","occurred_at":"2026-07-01T10:36:19Z","signature_id":"09a1acfc9ff3bef017ba8f327164609870cc787be902acbf6b7df883b58bf1b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072138226172170240","occurred_at":"2026-07-01T02:00:26Z","signature_id":"b4b60960c2dc2514808d53c4c7eb5cf8a0a347b89c3377d95e002fd8346d07b0","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}