{"meta":{"start_date":"2026-07-10","end_date":"2026-07-10","start_at":"2026-07-10T00:00:00Z","end_at":"2026-07-11T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T22:22:34Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":6,"failed_runs":5,"raw_occurrences":17,"matched_occurrences":15,"runs_affected":3},"rows":[{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-6hdcwz, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-6hdcwz in resource group rg-candidate-4-21-gtv29c, error: failed waiting for cluster=\"cluster-candidate-4-21-6hdcwz\" in resourcegroup=\"rg-candidate-4-21-gtv29c\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-6hdcwz in resource group rg-candidate-4-21-gtv29c, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":8,"runs_affected":3,"failed_runs":3,"run_impact_percent":50,"anchor_occurrences":8,"after_last_push_count":8,"trend_counts":[0,0,0,0,0,0,8],"trend_range":"2026-07-04..2026-07-10 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-08","2026-06-15","2026-06-22","2026-06-29"],"prior_runs_affected":59,"prior_last_seen_at":"2026-07-05T21:44:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000accde0\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-92v58x, error: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-92v58x\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-92v58x, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000accde0\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-92v58x, error: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-92v58x\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-92v58x, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-21-gtv29c/cluster-candidate-4-21-6hdcwz should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001116420\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-6hdcwz, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-6hdcwz in resource group rg-candidate-4-21-gtv29c, error: failed waiting for cluster=\"cluster-candidate-4-21-6hdcwz\" in resourcegroup=\"rg-candidate-4-21-gtv29c\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-6hdcwz in resource group rg-candidate-4-21-gtv29c, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-21-gtv29c/cluster-candidate-4-21-6hdcwz should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001116420\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-6hdcwz, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-6hdcwz in resource group rg-candidate-4-21-gtv29c, error: failed waiting for cluster=\"cluster-candidate-4-21-6hdcwz\" in resourcegroup=\"rg-candidate-4-21-gtv29c\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-6hdcwz in resource group rg-candidate-4-21-gtv29c, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:101]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0028aa5a0\u003e: \n    failed to create HCP cluster np-autoscale-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-whrdjm, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-whrdjm\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-whrdjm, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:101]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0028aa5a0\u003e: \n    failed to create HCP cluster np-autoscale-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-whrdjm, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-whrdjm\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-whrdjm, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2075625536775786496","occurred_at":"2026-07-10T16:57:45Z","signature_id":"09e43dc5e5e0445b72f0340be397ea83e802441cf7b41581c4eeb8fa6b70a3a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2075483182756532224","occurred_at":"2026-07-10T07:32:05Z","signature_id":"45f561d0cf9362d242ff0034dba71fa15fbd96d0a24576adceafc787311d4285","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2075399707978371072","occurred_at":"2026-07-10T02:00:24Z","signature_id":"146eecd7d8879c02a0dcfaaf8c796493a667f941eeaecbb68f189f6101d577d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2075399707978371072","occurred_at":"2026-07-10T02:00:24Z","signature_id":"1bc8566a51895e10d0236fed6f73e36fec9c73fd8064d59823a94a06d6297f92","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2075399707978371072","occurred_at":"2026-07-10T02:00:24Z","signature_id":"475b6d2b8821dc27667c60cadb73b1506ee49509fc8274b0235dc14f55a1d690","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2075399707978371072","occurred_at":"2026-07-10T02:00:24Z","signature_id":"b0a8b1a3e056748c18066b4208ff06e0877b7e70bdeb6e1d9f9f92b40910e732","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2075399707978371072","occurred_at":"2026-07-10T02:00:24Z","signature_id":"d1f09c4f69432a903a3f68c33513a4f6ba47410d2d37f06bd8a9a9a01a416865","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2075399707978371072","occurred_at":"2026-07-10T02:00:24Z","signature_id":"ff167fe9b2523becc3e8a415291cd834584a122a5dfeb2e9008200656af99a1f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"4e83fe1c3bb6daff1be6935129ccd1a1c67e021531f7a464980385c362ccc372","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-downgrade-fzxg7z-cqfgf2\", caused by: failed waiting for hcpCluster=\"np-downgrade-fzxg7z\" in resourcegroup=\"rg-np-downgrade-fzxg7z-cqfgf2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-downgrade-fzxg7z in resource group rg-np-downgrade-fzxg7z-cqfgf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-downgrade-fzxg7z\" in resourcegroup=\"rg-np-downgrade-fzxg7z-cqfgf2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-downgrade-fzxg7z in resource group rg-np-downgrade-fzxg7z-cqfgf2, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,0,5],"trend_range":"2026-07-04..2026-07-10 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc000fb8498\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-skip-minor-7wfv2w-wrn8zb\", caused by: failed waiting for hcpCluster=\"np-skip-minor-7wfv2w\" in resourcegroup=\"rg-np-skip-minor-7wfv2w-wrn8zb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-7wfv2w in resource group rg-np-skip-minor-7wfv2w-wrn8zb, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-skip-minor-7wfv2w\" in resourcegroup=\"rg-np-skip-minor-7wfv2w-wrn8zb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-7wfv2w in resource group rg-np-skip-minor-7wfv2w-wrn8zb, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc000fb8498\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-skip-minor-7wfv2w-wrn8zb\", caused by: failed waiting for hcpCluster=\"np-skip-minor-7wfv2w\" in resourcegroup=\"rg-np-skip-minor-7wfv2w-wrn8zb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-7wfv2w in resource group rg-np-skip-minor-7wfv2w-wrn8zb, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-skip-minor-7wfv2w\" in resourcegroup=\"rg-np-skip-minor-7wfv2w-wrn8zb\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-skip-minor-7wfv2w in resource group rg-np-skip-minor-7wfv2w-wrn8zb, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc00298e150\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-version-upgrade-n6xbpx-lm8v47\", caused by: failed waiting for hcpCluster=\"np-version-upgrade-cluster-n6xbpx\" in resourcegroup=\"rg-np-version-upgrade-n6xbpx-lm8v47\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-version-upgrade-cluster-n6xbpx in resource group rg-np-version-upgrade-n6xbpx-lm8v47, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-version-upgrade-cluster-n6xbpx\" in resourcegroup=\"rg-np-version-upgrade-n6xbpx-lm8v47\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-version-upgrade-cluster-n6xbpx in resource group rg-np-version-upgrade-n6xbpx-lm8v47, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc00298e150\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-version-upgrade-n6xbpx-lm8v47\", caused by: failed waiting for hcpCluster=\"np-version-upgrade-cluster-n6xbpx\" in resourcegroup=\"rg-np-version-upgrade-n6xbpx-lm8v47\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-version-upgrade-cluster-n6xbpx in resource group rg-np-version-upgrade-n6xbpx-lm8v47, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-version-upgrade-cluster-n6xbpx\" in resourcegroup=\"rg-np-version-upgrade-n6xbpx-lm8v47\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-version-upgrade-cluster-n6xbpx in resource group rg-np-version-upgrade-n6xbpx-lm8v47, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc000fc8390\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-downgrade-fzxg7z-cqfgf2\", caused by: failed waiting for hcpCluster=\"np-downgrade-fzxg7z\" in resourcegroup=\"rg-np-downgrade-fzxg7z-cqfgf2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-downgrade-fzxg7z in resource group rg-np-downgrade-fzxg7z-cqfgf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-downgrade-fzxg7z\" in resourcegroup=\"rg-np-downgrade-fzxg7z-cqfgf2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-downgrade-fzxg7z in resource group rg-np-downgrade-fzxg7z-cqfgf2, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc000fc8390\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-downgrade-fzxg7z-cqfgf2\", caused by: failed waiting for hcpCluster=\"np-downgrade-fzxg7z\" in resourcegroup=\"rg-np-downgrade-fzxg7z-cqfgf2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-downgrade-fzxg7z in resource group rg-np-downgrade-fzxg7z-cqfgf2, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-downgrade-fzxg7z\" in resourcegroup=\"rg-np-downgrade-fzxg7z-cqfgf2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-downgrade-fzxg7z in resource group rg-np-downgrade-fzxg7z-cqfgf2, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2075399707978371072","occurred_at":"2026-07-10T02:00:24Z","signature_id":"018b0a1c4bd3045bb50eb178378d1b4f96b95ceb3c412d18259816f3e0c20be5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2075399707978371072","occurred_at":"2026-07-10T02:00:24Z","signature_id":"5b6bdaefe211726ccaceedaa8d736c980ab6fd082307a01ae0c2d9419a8f0fa2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2075399707978371072","occurred_at":"2026-07-10T02:00:24Z","signature_id":"78a28aab0bf5d26c5b8c57395a9df867b496f883558f493a51bf9cea76f569dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2075399707978371072","occurred_at":"2026-07-10T02:00:24Z","signature_id":"95f39c0cd4a5412b87fd0d7b3c53a477f93b096686f4aa282d15a87e56b78a04","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2075399707978371072","occurred_at":"2026-07-10T02:00:24Z","signature_id":"9c5925d3d493788544cd3d27ab950d91b4c1c638cf0c50a9a7e19cbe7c0469cc","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-20 in resource group rg-np-version-upgrade-wjg86x-d529tv, error: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-wjg86x\" in resourcegroup=\"rg-np-version-upgrade-wjg86x-d529tv\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-04..2026-07-10 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-08","2026-06-15","2026-06-22","2026-06-29"],"prior_runs_affected":47,"prior_last_seen_at":"2026-07-05T23:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:151]: failed to create node pool npupgrade-4-20 with version 4.20.28\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002592660\u003e: \n    failed to create NodePool npupgrade-4-20, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-20 in resource group rg-np-version-upgrade-wjg86x-d529tv, error: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-wjg86x\" in resourcegroup=\"rg-np-version-upgrade-wjg86x-d529tv\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:151]: failed to create node pool npupgrade-4-20 with version 4.20.28\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002592660\u003e: \n    failed to create NodePool npupgrade-4-20, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-20 in resource group rg-np-version-upgrade-wjg86x-d529tv, error: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-wjg86x\" in resourcegroup=\"rg-np-version-upgrade-wjg86x-d529tv\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2075483182756532224","occurred_at":"2026-07-10T07:32:05Z","signature_id":"6bb8dda8c692800e3c2aff0d9195c986d7755ebe84bf5179be9d33f3d9f973a3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[0,0,0,1,0,0,1],"trend_range":"2026-07-04..2026-07-10 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-22"],"prior_runs_affected":5,"prior_last_seen_at":"2026-06-26T17:38:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-vcwc6l\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f82580\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-4lc9p.apps.aro.c9l5k9x0w5n8t1j.s25a.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-5-0-vcwc6l\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f82580\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-4lc9p.apps.aro.c9l5k9x0w5n8t1j.s25a.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2075483182756532224","occurred_at":"2026-07-10T07:32:05Z","signature_id":"f39aaf9c1303e3523ca9ba0de5dfd4ec6ed102b70870a2554a300bc1784de1f2","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}