{"meta":{"date":"2026-06-25","timezone":"UTC","generated_at":"2026-08-29T10:20:20Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":4,"failed_runs":3,"runs_with_occurrences":3,"runs_with_failure_pattern_matches":3,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":1,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2070280913878519808","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-25T23:00:08Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"30eda6a0d35040ddbc3b3a6d7201982d284b668564dde34d5b7fbe22e49afeec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2070280913878519808","occurred_at":"2026-06-25T23:00:08Z","failed_at":"e2e","signature_id":"515cfde894652e5d2bf00f29239afcc5194cafeffecf19a84ce0782115dd87a5","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-5-0-rbpvpr/cluster-nightly-5-0-664csl should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f8dbc0\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-664csl, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-664csl in resource group rg-nightly-5-0-rbpvpr, error: failed waiting for cluster=\"cluster-nightly-5-0-664csl\" in resourcegroup=\"rg-nightly-5-0-rbpvpr\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-664csl in resource group rg-nightly-5-0-rbpvpr, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-5-0-rbpvpr/cluster-nightly-5-0-664csl should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f8dbc0\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-664csl, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-664csl in resource group rg-nightly-5-0-rbpvpr, error: failed waiting for cluster=\"cluster-nightly-5-0-664csl\" in resourcegroup=\"rg-nightly-5-0-rbpvpr\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-664csl in resource group rg-nightly-5-0-rbpvpr, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-5-0-sb4vf2, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-sb4vf2 in resource group rg-candidate-5-0-jm2ghl, error: failed waiting for cluster=\"cluster-candidate-5-0-sb4vf2\" in resourcegroup=\"rg-candidate-5-0-jm2ghl\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-sb4vf2 in resource group rg-candidate-5-0-jm2ghl, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2070093862793121792","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-25T10:36:51Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"202d8db8b77a0dad7ef960c995dd741ac66babacb3b10a26a7c985c9f3e36ca9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2070093862793121792","occurred_at":"2026-06-25T10:36:51Z","failed_at":"e2e","signature_id":"74ec772d4b959eff888e30dd08c73ea14b8ae600fd7df1398191910740a3c94f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-jm2ghl/cluster-candidate-5-0-sb4vf2 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f8b110\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-sb4vf2, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-sb4vf2 in resource group rg-candidate-5-0-jm2ghl, error: failed waiting for cluster=\"cluster-candidate-5-0-sb4vf2\" in resourcegroup=\"rg-candidate-5-0-jm2ghl\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-sb4vf2 in resource group rg-candidate-5-0-jm2ghl, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-jm2ghl/cluster-candidate-5-0-sb4vf2 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f8b110\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-sb4vf2, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-sb4vf2 in resource group rg-candidate-5-0-jm2ghl, error: failed waiting for cluster=\"cluster-candidate-5-0-sb4vf2\" in resourcegroup=\"rg-candidate-5-0-jm2ghl\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-sb4vf2 in resource group rg-candidate-5-0-jm2ghl, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-5-0-sb4vf2, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-sb4vf2 in resource group rg-candidate-5-0-jm2ghl, error: failed waiting for cluster=\"cluster-candidate-5-0-sb4vf2\" in resourcegroup=\"rg-candidate-5-0-jm2ghl\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-sb4vf2 in resource group rg-candidate-5-0-jm2ghl, error: context deadline exceeded"}},{"row_id":"453ca160bab2bf461cc231debe16d1df2ad59bfcd7cc9520ece302df253035e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2070093862793121792","occurred_at":"2026-06-25T10:36:51Z","failed_at":"e2e","signature_id":"d677947a3befdfce6af5dffcbae4e0388155689f9c7b64fb547f4fba0c15273d","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:284] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001970168\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-57mhdb\" in resourcegroup=\"rg-np-version-upgrade-57mhdb-k6c55g\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:284] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001970168\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-57mhdb\" in resourcegroup=\"rg-np-version-upgrade-57mhdb-k6c55g\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f0e44476112ce4e675f687843407c03482d80d265239de5b2ddbd45c441a04e","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listswjsfq\" in resourcegroup=\"ea-list-556j9j\" to finish deleting: context canceled"}},{"row_id":"df27291a9d953eca18fe5fdbac78335861259d114c2f23dabb3b2c592c8c5342","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2070093862793121792","occurred_at":"2026-06-25T10:36:51Z","failed_at":"e2e","signature_id":"210f3648e7a374ef3b314d400049eee0704dddd28a091adf15a63b78a73994fe","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010800f0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-tvwh5g\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group complex-cilium-kv-tvwh5g, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010800f0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-tvwh5g\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group complex-cilium-kv-tvwh5g, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-5-0-sb4vf2, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-sb4vf2 in resource group rg-candidate-5-0-jm2ghl, error: failed waiting for cluster=\"cluster-candidate-5-0-sb4vf2\" in resourcegroup=\"rg-candidate-5-0-jm2ghl\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-sb4vf2 in resource group rg-candidate-5-0-jm2ghl, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","7f0e44476112ce4e675f687843407c03482d80d265239de5b2ddbd45c441a04e"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2069966174292021248","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-25T02:09:28Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d2330d0c505e15427ee52c180039eff17d36521ee269053cd79ec53aee4745a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2069966174292021248","occurred_at":"2026-06-25T02:09:28Z","failed_at":"e2e","signature_id":"1e784ab3fdf51282a18ce376758233e043051061c77fbde9af79b4fcd98b8e7e","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:284] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000fd6048\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listswjsfq\" in resourcegroup=\"ea-list-556j9j\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:284] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000fd6048\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listswjsfq\" in resourcegroup=\"ea-list-556j9j\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f0e44476112ce4e675f687843407c03482d80d265239de5b2ddbd45c441a04e","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listswjsfq\" in resourcegroup=\"ea-list-556j9j\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7f0e44476112ce4e675f687843407c03482d80d265239de5b2ddbd45c441a04e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2069963821681741824","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-06-25T02:00:07Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}