{"meta":{"start_date":"2026-06-04","end_date":"2026-06-04","start_at":"2026-06-04T00:00:00Z","end_at":"2026-06-05T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T11:44:45Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":6,"failed_runs":6,"raw_occurrences":15,"matched_occurrences":15,"runs_affected":6},"rows":[{"environment":"int","failure_pattern_id":"0ece011306dab74cd22dd19f1f3c89bbdb930a2321b61be59e4a6a90aae6e3eb","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-nightly-4-23-4h2ml7\" in resourcegroup=\"rg-nightly-4-23-btthl7\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-4h2ml7 in resource group rg-nightly-4-23-btthl7, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":7,"runs_affected":5,"failed_runs":5,"run_impact_percent":83.33333333333333,"anchor_occurrences":7,"after_last_push_count":7,"trend_counts":[0,0,0,0,0,0,7],"trend_range":"2026-05-29..2026-06-04 UTC","prior_windows_present":4,"prior_window_starts":["2026-05-04","2026-05-11","2026-05-18","2026-05-25"],"prior_runs_affected":30,"prior_last_seen_at":"2026-05-28T06:56:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:127]: Swift cluster rg-nightly-4-23-vs8qqq/cluster-nightly-4-23-vxm7h8 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013f2270\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-vxm7h8\" in resourcegroup=\"rg-nightly-4-23-vs8qqq\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-vxm7h8 in resource group rg-nightly-4-23-vs8qqq, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:127]: Swift cluster rg-nightly-4-23-vs8qqq/cluster-nightly-4-23-vxm7h8 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013f2270\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-vxm7h8\" in resourcegroup=\"rg-nightly-4-23-vs8qqq\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-vxm7h8 in resource group rg-nightly-4-23-vs8qqq, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:127]: Swift cluster rg-nightly-5-0-prsktx/cluster-nightly-5-0-mh65m5 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a50a50\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-mh65m5\" in resourcegroup=\"rg-nightly-5-0-prsktx\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-mh65m5 in resource group rg-nightly-5-0-prsktx, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:127]: Swift cluster rg-nightly-5-0-prsktx/cluster-nightly-5-0-mh65m5 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a50a50\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-mh65m5\" in resourcegroup=\"rg-nightly-5-0-prsktx\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-mh65m5 in resource group rg-nightly-5-0-prsktx, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:127]: Swift cluster rg-nightly-4-23-dkkxzs/cluster-nightly-4-23-vzchhr should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010141e0\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-vzchhr\" in resourcegroup=\"rg-nightly-4-23-dkkxzs\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-vzchhr in resource group rg-nightly-4-23-dkkxzs, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:127]: Swift cluster rg-nightly-4-23-dkkxzs/cluster-nightly-4-23-vzchhr should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010141e0\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-vzchhr\" in resourcegroup=\"rg-nightly-4-23-dkkxzs\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-vzchhr in resource group rg-nightly-4-23-dkkxzs, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062652262928355328","occurred_at":"2026-06-04T21:46:36Z","signature_id":"2bb5548e63b0e34a4b912c3246043e3a94403bb8126da226b2a96ed2964ac29a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062652262928355328","occurred_at":"2026-06-04T21:46:36Z","signature_id":"6224ee3310f53bd2672d87436d0d045863d834789a61cdbd52d886ea3877c13e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062580850595205120","occurred_at":"2026-06-04T17:02:50Z","signature_id":"c952e2027f98a67de8f5e619c6e01631ac837ecd30dad94fd10be30fafacd170","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062534999151742976","occurred_at":"2026-06-04T14:00:18Z","signature_id":"48cd8301d6430030c45e54f67023949338e411defe23443c9c96a9170f95baf8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062470512495300608","occurred_at":"2026-06-04T09:44:23Z","signature_id":"28384a2cdedbb6881622910e4583cb946eaed78328a8bdb5cd302b65cea81532","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062420225306923008","occurred_at":"2026-06-04T06:24:34Z","signature_id":"2e92ec4aa6dd7f511cf1ea487d96bedffccc53b08029ba9e14d1d42c9094021a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062420225306923008","occurred_at":"2026-06-04T06:24:34Z","signature_id":"d14655f89322ae89f91d58cfe4a59ba4f6a5192a456bede2af04b010f419f81e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"a880b7dbbe8686372b606e0212fd3742bc75c3c0a9c545bae9481b243ed0f55b","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-4bzj9p in resource group rg-candidate-4-20-xdxq7x, error: failed waiting for nodepool=\"np-4bzj9p\" for cluster \"cluster-candidate-4-20-4bzj9p\" in resourcegroup=\"rg-candidate-4-20-xdxq7x\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":6,"runs_affected":4,"failed_runs":4,"run_impact_percent":66.66666666666667,"anchor_occurrences":6,"after_last_push_count":10,"trend_counts":[0,0,0,0,0,4,6],"trend_range":"2026-05-29..2026-06-04 UTC","prior_windows_present":1,"prior_window_starts":["2026-05-04"],"prior_runs_affected":7,"prior_last_seen_at":"2026-05-09T02:53:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:193]: failed to create node pool \"np-x9tqjq\" for cluster \"cluster-candidate-4-22-x9tqjq\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010e98c0\u003e: \n    failed to create NodePool np-x9tqjq, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-x9tqjq in resource group rg-candidate-4-22-qvtzjw, error: failed waiting for nodepool=\"np-x9tqjq\" for cluster \"cluster-candidate-4-22-x9tqjq\" in resourcegroup=\"rg-candidate-4-22-qvtzjw\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:193]: failed to create node pool \"np-x9tqjq\" for cluster \"cluster-candidate-4-22-x9tqjq\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010e98c0\u003e: \n    failed to create NodePool np-x9tqjq, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-x9tqjq in resource group rg-candidate-4-22-qvtzjw, error: failed waiting for nodepool=\"np-x9tqjq\" for cluster \"cluster-candidate-4-22-x9tqjq\" in resourcegroup=\"rg-candidate-4-22-qvtzjw\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:193]: failed to create node pool \"np-8chkd7\" for cluster \"cluster-candidate-4-21-8chkd7\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e67620\u003e: \n    failed to create NodePool np-8chkd7, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-8chkd7 in resource group rg-candidate-4-21-dccmdv, error: failed waiting for nodepool=\"np-8chkd7\" for cluster \"cluster-candidate-4-21-8chkd7\" in resourcegroup=\"rg-candidate-4-21-dccmdv\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:193]: failed to create node pool \"np-8chkd7\" for cluster \"cluster-candidate-4-21-8chkd7\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e67620\u003e: \n    failed to create NodePool np-8chkd7, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-8chkd7 in resource group rg-candidate-4-21-dccmdv, error: failed waiting for nodepool=\"np-8chkd7\" for cluster \"cluster-candidate-4-21-8chkd7\" in resourcegroup=\"rg-candidate-4-21-dccmdv\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:193]: failed to create node pool \"np-zxj94k\" for cluster \"cluster-candidate-4-21-zxj94k\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001431050\u003e: \n    failed to create NodePool np-zxj94k, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-zxj94k in resource group rg-candidate-4-21-n49q2q, error: failed waiting for nodepool=\"np-zxj94k\" for cluster \"cluster-candidate-4-21-zxj94k\" in resourcegroup=\"rg-candidate-4-21-n49q2q\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:193]: failed to create node pool \"np-zxj94k\" for cluster \"cluster-candidate-4-21-zxj94k\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001431050\u003e: \n    failed to create NodePool np-zxj94k, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-zxj94k in resource group rg-candidate-4-21-n49q2q, error: failed waiting for nodepool=\"np-zxj94k\" for cluster \"cluster-candidate-4-21-zxj94k\" in resourcegroup=\"rg-candidate-4-21-n49q2q\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062652262928355328","occurred_at":"2026-06-04T21:46:36Z","signature_id":"1b3b763b51b4c5e96014cadcb0c40de92a6fa877a84a80bea0f8af32b9e75499","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062652262928355328","occurred_at":"2026-06-04T21:46:36Z","signature_id":"7a89427826dfe0adfad300b92dd6b1baae7e111d9e0627a5e00d87def32db008","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062580850595205120","occurred_at":"2026-06-04T17:02:50Z","signature_id":"c6cde17409ec591dbd2573278494006529844ed4c926f575b920e72aa9f07ae0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062534999151742976","occurred_at":"2026-06-04T14:00:18Z","signature_id":"0d758579abe5f0a3574fdb25df8064f0ca35266619c9c2f209ea6b470c451c03","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062470512495300608","occurred_at":"2026-06-04T09:44:23Z","signature_id":"615674b68b082350bc3b2df030a7c4356d4ecb17321232d546f1b6ab61fca664","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062470512495300608","occurred_at":"2026-06-04T09:44:23Z","signature_id":"c8c3091611e473bc0acfeb29d661ad54c9be25a6a63962e9ca91591f8411e5e8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"52d9258f03a60774b49eaa70a514574e25b40d0eeb6ddfa13f252e24164f6be2","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-xbl4zl\" in resourcegroup=\"rg-candidate-4-22-9bdr9k\" to finish deleting: context canceled","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-05-29..2026-06-04 UTC","prior_windows_present":3,"prior_window_starts":["2026-05-11","2026-05-18","2026-05-25"],"prior_runs_affected":12,"prior_last_seen_at":"2026-05-27T03:09:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:278] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00151a300\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-6fzlhg\" in resourcegroup=\"rg-np-version-upgrade-6fzlhg-lfwn97\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:278] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00151a300\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-6fzlhg\" in resourcegroup=\"rg-np-version-upgrade-6fzlhg-lfwn97\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:278] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0015f22e8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-xbl4zl\" in resourcegroup=\"rg-candidate-4-22-9bdr9k\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:278] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0015f22e8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-xbl4zl\" in resourcegroup=\"rg-candidate-4-22-9bdr9k\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062341047589015552","occurred_at":"2026-06-04T01:09:56Z","signature_id":"a94c21a5d1649b93f1d5c878957201254b1da65fa35f89695cb669006291f71b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062341047589015552","occurred_at":"2026-06-04T01:09:56Z","signature_id":"fa042af2d74eb7b3d10ed35f117a7634b13a214c611777366b34ddf21652e865","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}