{"meta":{"date":"2026-06-28","timezone":"UTC","generated_at":"2026-08-28T06:57:08Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":3,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":1,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2071368141530730496","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-28T23:00:23Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"3c212eaf1554cf4837b04cacd99050bf4e5ab8be6032163cca5418d44131811c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2071368141530730496","occurred_at":"2026-06-28T23:00:23Z","failed_at":"e2e","signature_id":"441b6d97911f886e0224e335cee97dd32f5ac764d1ffb467f983f3add45058f8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-qt62hg\" for cluster \"cluster-nightly-4-22-qt62hg\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001963170\u003e: \n    failed to create NodePool np-qt62hg, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-qt62hg in resource group rg-nightly-4-22-89k865, error: failed waiting for nodepool=\"np-qt62hg\" for cluster \"cluster-nightly-4-22-qt62hg\" in resourcegroup=\"rg-nightly-4-22-89k865\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-qt62hg\" for cluster \"cluster-nightly-4-22-qt62hg\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001963170\u003e: \n    failed to create NodePool np-qt62hg, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-qt62hg in resource group rg-nightly-4-22-89k865, error: failed waiting for nodepool=\"np-qt62hg\" for cluster \"cluster-nightly-4-22-qt62hg\" in resourcegroup=\"rg-nightly-4-22-89k865\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-noaz in resource group np-autoscaling-f2qq9q, error: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-f2qq9q\" to finish creating: context deadline exceeded"}},{"row_id":"ddd4efe840dfe6d845231f17968ae2324c7eac7fd485fa2e95dc9ab113937120","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2071368141530730496","occurred_at":"2026-06-28T23:00:23Z","failed_at":"e2e","signature_id":"b47c33be2132af3d48dfa29add64496488d3c05a3948bf68e839d294e89f1e9f","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00130e300\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-nfhflv\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00130e300\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-nfhflv\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f0e44476112ce4e675f687843407c03482d80d265239de5b2ddbd45c441a04e","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-nfhflv\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7f0e44476112ce4e675f687843407c03482d80d265239de5b2ddbd45c441a04e","941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2071331624070418432","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-28T20:35:17Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"061c7faaac9c3777b39fff760c3fba26a813baa8b093b139f42654ff313cdb07","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2071331624070418432","occurred_at":"2026-06-28T20:35:17Z","failed_at":"e2e","signature_id":"827ec00a4ccbc057a79d910fdc1accdc3d7590478989dd3b9f81d7a8975b5b82","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:214]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a33ad0\u003e: \n    failed to create NodePool autoscale-noaz, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-noaz in resource group np-autoscaling-f2qq9q, error: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-f2qq9q\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:214]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a33ad0\u003e: \n    failed to create NodePool autoscale-noaz, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-noaz in resource group np-autoscaling-f2qq9q, error: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-f2qq9q\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-noaz in resource group np-autoscaling-f2qq9q, error: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-f2qq9q\" to finish creating: context deadline exceeded"}},{"row_id":"09679a126e83889f6794bf6507ebe3bacdb6e73d8d3339d4185869f913d5c2dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2071331624070418432","occurred_at":"2026-06-28T20:35:17Z","failed_at":"e2e","signature_id":"65b7a10b456bbd873078f4fd4c9d0ab632c9dea1e8097b09eb788630e5be2fce","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-wg5rzb/cluster-candidate-5-0-4q5ggx should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014f0720\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-4q5ggx, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-4q5ggx in resource group rg-candidate-5-0-wg5rzb, error: failed waiting for cluster=\"cluster-candidate-5-0-4q5ggx\" in resourcegroup=\"rg-candidate-5-0-wg5rzb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-4q5ggx in resource group rg-candidate-5-0-wg5rzb, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-wg5rzb/cluster-candidate-5-0-4q5ggx should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014f0720\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-4q5ggx, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-4q5ggx in resource group rg-candidate-5-0-wg5rzb, error: failed waiting for cluster=\"cluster-candidate-5-0-4q5ggx\" in resourcegroup=\"rg-candidate-5-0-wg5rzb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-4q5ggx in resource group rg-candidate-5-0-wg5rzb, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-22-wmxsmt, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-wmxsmt in resource group rg-candidate-4-22-7bcwz9, error: failed waiting for cluster=\"cluster-candidate-4-22-wmxsmt\" in resourcegroup=\"rg-candidate-4-22-7bcwz9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-wmxsmt in resource group rg-candidate-4-22-7bcwz9, error: context deadline exceeded"}},{"row_id":"435efa4bce28e1e2b2408b38de01946809a58d48c4a1247bf493b1539110d7d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2071331624070418432","occurred_at":"2026-06-28T20:35:17Z","failed_at":"e2e","signature_id":"3f7243f8e285c3f9ecbc6d7fad6b00889350dcd67ac766d3ebda3bbe9ccb61f1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-7bcwz9/cluster-candidate-4-22-wmxsmt should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a1c1b0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-wmxsmt, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-wmxsmt in resource group rg-candidate-4-22-7bcwz9, error: failed waiting for cluster=\"cluster-candidate-4-22-wmxsmt\" in resourcegroup=\"rg-candidate-4-22-7bcwz9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-wmxsmt in resource group rg-candidate-4-22-7bcwz9, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-7bcwz9/cluster-candidate-4-22-wmxsmt should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a1c1b0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-wmxsmt, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-wmxsmt in resource group rg-candidate-4-22-7bcwz9, error: failed waiting for cluster=\"cluster-candidate-4-22-wmxsmt\" in resourcegroup=\"rg-candidate-4-22-7bcwz9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-wmxsmt in resource group rg-candidate-4-22-7bcwz9, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-22-wmxsmt, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-wmxsmt in resource group rg-candidate-4-22-7bcwz9, error: failed waiting for cluster=\"cluster-candidate-4-22-wmxsmt\" in resourcegroup=\"rg-candidate-4-22-7bcwz9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-wmxsmt in resource group rg-candidate-4-22-7bcwz9, error: context deadline exceeded"}},{"row_id":"a82348af362d912e9225badc18f8551d168b1346349c7c11cc6e0ee042c72d91","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2071331624070418432","occurred_at":"2026-06-28T20:35:17Z","failed_at":"e2e","signature_id":"7ff6a8cd5d1a64c38ebd4d68a4adbd5cffeffe8d922069aab0ab21e5ef680c2c","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013be438\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-nr68nw\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013be438\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-nr68nw\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f0e44476112ce4e675f687843407c03482d80d265239de5b2ddbd45c441a04e","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-nfhflv\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","7f0e44476112ce4e675f687843407c03482d80d265239de5b2ddbd45c441a04e","941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2071051048939687936","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-06-28T02:00:22Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}