{"meta":{"start_date":"2026-06-26","end_date":"2026-06-26","start_at":"2026-06-26T00:00:00Z","end_at":"2026-06-27T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T08:27:26Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":6,"failed_runs":4,"raw_occurrences":4,"matched_occurrences":4,"runs_affected":4},"rows":[{"environment":"int","failure_pattern_id":"0ece011306dab74cd22dd19f1f3c89bbdb930a2321b61be59e4a6a90aae6e3eb","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-rb9trs\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group complex-cilium-kv-rb9trs, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":33.333333333333336,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-06-20..2026-06-26 UTC","prior_windows_present":4,"prior_window_starts":["2026-05-25","2026-06-01","2026-06-08","2026-06-15"],"prior_runs_affected":41,"prior_last_seen_at":"2026-06-19T16:37:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000764120\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-rb9trs\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group complex-cilium-kv-rb9trs, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000764120\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-rb9trs\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group complex-cilium-kv-rb9trs, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:102]: failed to create HCP cluster \"cluster-zstream-4-22-sv8tjk\" with version 4.22.1 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e4c060\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-sv8tjk, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-sv8tjk in resource group rg-zstream-upgrade-4-22-l997c2, error: failed waiting for cluster=\"cluster-zstream-4-22-sv8tjk\" in resourcegroup=\"rg-zstream-upgrade-4-22-l997c2\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-sv8tjk in resource group rg-zstream-upgrade-4-22-l997c2, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:102]: failed to create HCP cluster \"cluster-zstream-4-22-sv8tjk\" with version 4.22.1 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e4c060\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-sv8tjk, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-sv8tjk in resource group rg-zstream-upgrade-4-22-l997c2, error: failed waiting for cluster=\"cluster-zstream-4-22-sv8tjk\" in resourcegroup=\"rg-zstream-upgrade-4-22-l997c2\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-sv8tjk in resource group rg-zstream-upgrade-4-22-l997c2, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2070615009901154304","occurred_at":"2026-06-26T21:07:43Z","signature_id":"8aaba981b0d935d308ba896a614cacb0180bba87ec2aae03f4e3a33d3030b8ce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2070361227929849856","occurred_at":"2026-06-26T04:19:16Z","signature_id":"d57760d7484dc047ed0150b68be4f511b0f12dae0ee346efa7c42f164044b1d8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"a880b7dbbe8686372b606e0212fd3742bc75c3c0a9c545bae9481b243ed0f55b","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-l9xh4z, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-l9xh4z\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-06-20..2026-06-26 UTC","prior_windows_present":3,"prior_window_starts":["2026-06-01","2026-06-08","2026-06-15"],"prior_runs_affected":20,"prior_last_seen_at":"2026-06-18T08:29:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001c7ad50\u003e: \n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-l9xh4z, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-l9xh4z\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001c7ad50\u003e: \n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-l9xh4z, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-l9xh4z\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2070312313818189824","occurred_at":"2026-06-26T01:04:54Z","signature_id":"0fcfbf30750374c36ef4be303f48c21ce3f3d66584909d19998c1e78ceba63b5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"77fb076d32302f92d929786f345bdad3139a0df750404c0633de671aa620060b","failure_pattern":"x509: certificate signed by unknown authority","search_query":"x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,1,0,0,0,0,1],"trend_range":"2026-06-20..2026-06-26 UTC","prior_windows_present":2,"prior_window_starts":["2026-06-08","2026-06-15"],"prior_runs_affected":3,"prior_last_seen_at":"2026-06-21T22:10:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:167]: Timed out after 600.132s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:167]: Timed out after 600.132s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2070533078811414528","occurred_at":"2026-06-26T15:42:09Z","signature_id":"c887bcbd9fb9e55b68ddc0f8676b9902e4e98b0144f05bad1a198677d96de9f5","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}