{"meta":{"start_date":"2026-06-10","end_date":"2026-06-10","start_at":"2026-06-10T00:00:00Z","end_at":"2026-06-11T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T11:59:55Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":5,"failed_runs":2,"raw_occurrences":4,"matched_occurrences":4,"runs_affected":2},"rows":[{"environment":"int","failure_pattern_id":"a880b7dbbe8686372b606e0212fd3742bc75c3c0a9c545bae9481b243ed0f55b","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-5n766t, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-5n766t\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":40,"anchor_occurrences":3,"after_last_push_count":16,"trend_counts":[6,0,0,0,0,3,3],"trend_range":"2026-06-04..2026-06-10 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-01"],"prior_runs_affected":11,"prior_last_seen_at":"2026-06-04T21:46:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-swgddz\" for cluster \"cluster-candidate-4-21-swgddz\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ec4d80\u003e: \n    failed to create NodePool np-swgddz, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-swgddz in resource group rg-candidate-4-21-zncqmm, error: failed waiting for nodepool=\"np-swgddz\" for cluster \"cluster-candidate-4-21-swgddz\" in resourcegroup=\"rg-candidate-4-21-zncqmm\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-swgddz\" for cluster \"cluster-candidate-4-21-swgddz\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ec4d80\u003e: \n    failed to create NodePool np-swgddz, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-swgddz in resource group rg-candidate-4-21-zncqmm, error: failed waiting for nodepool=\"np-swgddz\" for cluster \"cluster-candidate-4-21-swgddz\" in resourcegroup=\"rg-candidate-4-21-zncqmm\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000f06390\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-5n766t, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-5n766t\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000f06390\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-5n766t, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-5n766t\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:136]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a177a0\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-cv5cw9, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-cv5cw9\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:136]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a177a0\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-cv5cw9, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-cv5cw9\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2064803737464999936","occurred_at":"2026-06-10T20:15:47Z","signature_id":"53dfd2ee39d64d2bc9b69cbf05d23a74e185ff0bbd83c410f90dca0715e6a347","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2064701571148550144","occurred_at":"2026-06-10T13:29:39Z","signature_id":"b4cc1ee449ed2b41c14c3edd6295fa4c14e370c4d4a5835bb721f25a8dd11036","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2064701571148550144","occurred_at":"2026-06-10T13:29:39Z","signature_id":"ec963c785096f2e47a00fe856e71726e570bc56b3da9c161f998ee7aab8bc50f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"0ece011306dab74cd22dd19f1f3c89bbdb930a2321b61be59e4a6a90aae6e3eb","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-nightly-4-23-qrf77g\" in resourcegroup=\"rg-nightly-4-23-d4vw2m\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-qrf77g in resource group rg-nightly-4-23-d4vw2m, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-06-04..2026-06-10 UTC","prior_windows_present":4,"prior_window_starts":["2026-05-11","2026-05-18","2026-05-25","2026-06-01"],"prior_runs_affected":28,"prior_last_seen_at":"2026-06-05T18:49:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-d4vw2m/cluster-nightly-4-23-qrf77g should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00140ec60\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-qrf77g\" in resourcegroup=\"rg-nightly-4-23-d4vw2m\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-qrf77g in resource group rg-nightly-4-23-d4vw2m, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-d4vw2m/cluster-nightly-4-23-qrf77g should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00140ec60\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-qrf77g\" in resourcegroup=\"rg-nightly-4-23-d4vw2m\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-qrf77g in resource group rg-nightly-4-23-d4vw2m, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2064803737464999936","occurred_at":"2026-06-10T20:15:47Z","signature_id":"314476787b94ab22e694cb6a10a87966c94317ee70f5f4585c95e660227d4f89","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}