{"meta":{"start_date":"2026-05-04","end_date":"2026-05-10","start_at":"2026-05-04T00:00:00Z","end_at":"2026-05-11T00:00:00Z","timezone":"UTC","generated_at":"2026-08-27T07:19:53Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":17,"failed_runs":3,"raw_occurrences":4,"matched_occurrences":4,"runs_affected":3},"rows":[{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-7vj672 in resource group rg-zstream-upgrade-4-20-826lfb, error: failed waiting for cluster=\"cluster-zstream-4-20-7vj672\" in resourcegroup=\"rg-zstream-upgrade-4-20-826lfb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-7vj672 in resource group rg-zstream-upgrade-4-20-826lfb, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":11.764705882352942,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,2,0,1,0,0,0,0],"trend_range":"2026-05-03..2026-05-10 UTC","prior_windows_present":3,"prior_window_starts":["2026-04-13","2026-04-20","2026-04-27"],"prior_runs_affected":10,"prior_last_seen_at":"2026-04-29T02:00:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015f2540\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-7vj672, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-7vj672 in resource group rg-zstream-upgrade-4-20-826lfb, error: failed waiting for cluster=\"cluster-zstream-4-20-7vj672\" in resourcegroup=\"rg-zstream-upgrade-4-20-826lfb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-7vj672 in resource group rg-zstream-upgrade-4-20-826lfb, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015f2540\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-7vj672, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-7vj672 in resource group rg-zstream-upgrade-4-20-826lfb, error: failed waiting for cluster=\"cluster-zstream-4-20-7vj672\" in resourcegroup=\"rg-zstream-upgrade-4-20-826lfb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-7vj672 in resource group rg-zstream-upgrade-4-20-826lfb, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:130]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0017111d0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-pv5rlg, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-pv5rlg in resource group rg-np-version-upgrade-pv5rlg-jc6hhn, error: failed waiting for cluster=\"np-version-upgrade-cluster-pv5rlg\" in resourcegroup=\"rg-np-version-upgrade-pv5rlg-jc6hhn\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-pv5rlg in resource group rg-np-version-upgrade-pv5rlg-jc6hhn, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:130]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0017111d0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-pv5rlg, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-pv5rlg in resource group rg-np-version-upgrade-pv5rlg-jc6hhn, error: failed waiting for cluster=\"np-version-upgrade-cluster-pv5rlg\" in resourcegroup=\"rg-np-version-upgrade-pv5rlg-jc6hhn\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-pv5rlg in resource group rg-np-version-upgrade-pv5rlg-jc6hhn, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012a63f0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-cdn7lv, error: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-cdn7lv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-cdn7lv, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012a63f0\u003e: \n    failed to create HCP cluster cluster-np-labels-taints, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-cdn7lv, error: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-cdn7lv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-cdn7lv, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2051844452711927808","occurred_at":"2026-05-06T02:00:13Z","signature_id":"59da621455ea697d98f67cbfcf50b10bd4795f773ad7fd0de36b6944fec3a680","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2051387205661757440","occurred_at":"2026-05-04T19:43:17Z","signature_id":"128d55783564a9d5f82db9a0e7abb7e8c5176cdbabbe50290b56015075e3e389","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2051387205661757440","occurred_at":"2026-05-04T19:43:17Z","signature_id":"5bd5f750870a870aa29a5bef34080c867b9117b386ed19ea01f1199e0a1c7a18","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5.882352941176471,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-05-03..2026-05-10 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:168]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000f044c0\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-tkwcr.apps.aro.cilium-cl.ufap.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:168]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000f044c0\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-tkwcr.apps.aro.cilium-cl.ufap.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2052689559451340800","occurred_at":"2026-05-08T09:58:22Z","signature_id":"2489a05819e15cac8945138f6e45d61afe406d0696cd6e846f27406637ba7131","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}