{"meta":{"start_date":"2026-06-17","end_date":"2026-06-17","start_at":"2026-06-17T00:00:00Z","end_at":"2026-06-18T00:00:00Z","timezone":"UTC","generated_at":"2026-08-27T02:50:19Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":6,"failed_runs":3,"raw_occurrences":17,"matched_occurrences":16,"runs_affected":2},"rows":[{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-nmbslb, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-nmbslb in resource group rg-candidate-4-20-ppvxkq, error: failed waiting for cluster=\"cluster-candidate-4-20-nmbslb\" in resourcegroup=\"rg-candidate-4-20-ppvxkq\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-nmbslb in resource group rg-candidate-4-20-ppvxkq, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":15,"runs_affected":2,"failed_runs":2,"run_impact_percent":33.333333333333336,"anchor_occurrences":15,"after_last_push_count":15,"trend_counts":[0,0,0,0,0,0,15],"trend_range":"2026-06-11..2026-06-17 UTC","prior_windows_present":4,"prior_window_starts":["2026-05-18","2026-05-25","2026-06-01","2026-06-08"],"prior_runs_affected":29,"prior_last_seen_at":"2026-06-13T23:00:09Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-4mmpsx/cluster-candidate-5-0-b42g4w should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00159aa20\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-b42g4w, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-b42g4w in resource group rg-candidate-5-0-4mmpsx, error: failed waiting for cluster=\"cluster-candidate-5-0-b42g4w\" in resourcegroup=\"rg-candidate-5-0-4mmpsx\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-b42g4w in resource group rg-candidate-5-0-4mmpsx, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-4mmpsx/cluster-candidate-5-0-b42g4w should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00159aa20\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-b42g4w, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-b42g4w in resource group rg-candidate-5-0-4mmpsx, error: failed waiting for cluster=\"cluster-candidate-5-0-b42g4w\" in resourcegroup=\"rg-candidate-5-0-4mmpsx\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-b42g4w in resource group rg-candidate-5-0-4mmpsx, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-l7qpx5/cluster-candidate-5-0-njtt9w should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012d6690\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-njtt9w, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-njtt9w in resource group rg-candidate-5-0-l7qpx5, error: failed waiting for cluster=\"cluster-candidate-5-0-njtt9w\" in resourcegroup=\"rg-candidate-5-0-l7qpx5\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-njtt9w in resource group rg-candidate-5-0-l7qpx5, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-l7qpx5/cluster-candidate-5-0-njtt9w should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012d6690\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-njtt9w, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-njtt9w in resource group rg-candidate-5-0-l7qpx5, error: failed waiting for cluster=\"cluster-candidate-5-0-njtt9w\" in resourcegroup=\"rg-candidate-5-0-l7qpx5\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-njtt9w in resource group rg-candidate-5-0-l7qpx5, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:83]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e7f500\u003e: \n    failed to create HCP cluster cilium-cl, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cilium-cl in resource group cni-cilium-858gdp, error: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-858gdp\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cilium-cl in resource group cni-cilium-858gdp, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:83]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e7f500\u003e: \n    failed to create HCP cluster cilium-cl, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cilium-cl in resource group cni-cilium-858gdp, error: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-858gdp\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cilium-cl in resource group cni-cilium-858gdp, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2067109479970770944","occurred_at":"2026-06-17T04:57:58Z","signature_id":"342e2aa7a5e0f9ab06df8f5b07feaa4df986eddb1453d3e3c673a0c09970fb08","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2067035760380350464","occurred_at":"2026-06-17T00:05:03Z","signature_id":"03a0b476943b6d72615b9c1af6a11a4301cf6abb5879bdb16b53fe72e98c8a15","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2067035760380350464","occurred_at":"2026-06-17T00:05:03Z","signature_id":"29715bbfffd0f75e1c991259a2854582448c83863c2cd9cb22e840c29b744a8c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2067035760380350464","occurred_at":"2026-06-17T00:05:03Z","signature_id":"4e21acd55d346b66a43987ff9f5d66b5cabaf2d0ee965a68fcb56a2d0c300610","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2067035760380350464","occurred_at":"2026-06-17T00:05:03Z","signature_id":"5b63446266cdb340e71d116017cd550bc13ced25473a1118bcc6f847aea4a2b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2067035760380350464","occurred_at":"2026-06-17T00:05:03Z","signature_id":"6a7af4b57f3eb2638914d8429cdcf648131d919e96f99c1920b319eb28c6d006","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2067035760380350464","occurred_at":"2026-06-17T00:05:03Z","signature_id":"6e3b1570e911fb8533f23dca85fc89725d6a587900c6484c74f58f98b83df409","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2067035760380350464","occurred_at":"2026-06-17T00:05:03Z","signature_id":"718b5a382670ad1e538715f8f823fea822e28ca6968b1b57e45f226d29cafda1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2067035760380350464","occurred_at":"2026-06-17T00:05:03Z","signature_id":"724d1b9af379b2c9f6f83fb6e5e1d3e3997a4a92d1269c64d784fbd5be93b2f7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2067035760380350464","occurred_at":"2026-06-17T00:05:03Z","signature_id":"9dc452e294e3a6c65db76b47df2f0a626537bef63930f8a6e61a2ffb2812e827","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2067035760380350464","occurred_at":"2026-06-17T00:05:03Z","signature_id":"9eaa4e61392c79782dce33d5d40680b5734ef9f7f8ef91257af38a0931c6a583","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2067035760380350464","occurred_at":"2026-06-17T00:05:03Z","signature_id":"a5043110bc3a51eecc84a98d96eed9f6cbe7a473812f6bd37dea8ab13488c353","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2067035760380350464","occurred_at":"2026-06-17T00:05:03Z","signature_id":"dec87334b1f8b163371231b037449aadab735032cc68337aef5c81300188f060","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2067035760380350464","occurred_at":"2026-06-17T00:05:03Z","signature_id":"f1d8abe8cadcf37ebb0a1e2818fdfb3a5c3d680ee75f12f4c9809b4bd49d2751","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2067035760380350464","occurred_at":"2026-06-17T00:05:03Z","signature_id":"fda8c09be817f1165ac1a3c00bbcc00f67da3227dc7e76b6539fd66ab9398658","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"0fba11a4b2b48d674dab9e0772d507b27de853b49d3a03fba0eb386b68b7044f","failure_pattern":"failed to wait for first cluster \"\u003ccluster\u003e\" to complete creation (timeout '20.000000' minutes)","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-06-11..2026-06-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '20.000000' minutes)\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2067035760380350464","occurred_at":"2026-06-17T00:05:03Z","signature_id":"fa82b432fd7a36953b4c71f37791bced4d6db1832370f07a5c77508bfa36ef7e","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}