{"meta":{"start_date":"2026-06-03","end_date":"2026-06-03","start_at":"2026-06-03T00:00:00Z","end_at":"2026-06-04T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T12:38:51Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":5,"failed_runs":4,"raw_occurrences":8,"matched_occurrences":8,"runs_affected":4},"rows":[{"environment":"int","failure_pattern_id":"0ece011306dab74cd22dd19f1f3c89bbdb930a2321b61be59e4a6a90aae6e3eb","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-nightly-4-23-99498l\" in resourcegroup=\"rg-nightly-4-23-zkrlm6\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-99498l in resource group rg-nightly-4-23-zkrlm6, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":80,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-05-28..2026-06-03 UTC","prior_windows_present":4,"prior_window_starts":["2026-05-04","2026-05-11","2026-05-18","2026-05-25"],"prior_runs_affected":25,"prior_last_seen_at":"2026-05-28T06:56:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:127]: Swift cluster rg-nightly-4-23-zkrlm6/cluster-nightly-4-23-99498l should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011dec00\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-99498l\" in resourcegroup=\"rg-nightly-4-23-zkrlm6\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-99498l in resource group rg-nightly-4-23-zkrlm6, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:127]: Swift cluster rg-nightly-4-23-zkrlm6/cluster-nightly-4-23-99498l should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011dec00\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-99498l\" in resourcegroup=\"rg-nightly-4-23-zkrlm6\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-99498l in resource group rg-nightly-4-23-zkrlm6, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:127]: Swift cluster rg-nightly-5-0-wrwpzj/cluster-nightly-5-0-jjcdjk should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d1a600\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-jjcdjk\" in resourcegroup=\"rg-nightly-5-0-wrwpzj\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-jjcdjk in resource group rg-nightly-5-0-wrwpzj, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:127]: Swift cluster rg-nightly-5-0-wrwpzj/cluster-nightly-5-0-jjcdjk should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d1a600\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-jjcdjk\" in resourcegroup=\"rg-nightly-5-0-wrwpzj\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-jjcdjk in resource group rg-nightly-5-0-wrwpzj, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a33380\u003e: \n    failed to create HCP cluster oidc-wi-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-9pfbmq, error: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-9pfbmq\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-9pfbmq, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a33380\u003e: \n    failed to create HCP cluster oidc-wi-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-9pfbmq, error: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-9pfbmq\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster oidc-wi-cluster in resource group oidc-wi-9pfbmq, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062289277621899264","occurred_at":"2026-06-03T21:44:11Z","signature_id":"5862d23853fed32642752f569261d08599bb52d873e8b878975cc73829a5159e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062239668270796800","occurred_at":"2026-06-03T18:27:06Z","signature_id":"bfc6e840c8214e25c8de1cbab56aa54c683ee7b4b12dda2e43000353f183050f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062083589301866496","occurred_at":"2026-06-03T08:06:53Z","signature_id":"2bf5fed22f4a3a0f98728f13d10330567c8df98a3398daca124924fec09d09f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062031569979183104","occurred_at":"2026-06-03T04:40:11Z","signature_id":"8738a00b0e9c6737aae3fd774f2f72ff01f1df67edcaee01ab6b2ec886c331f5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"a880b7dbbe8686372b606e0212fd3742bc75c3c0a9c545bae9481b243ed0f55b","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-4gllqb in resource group rg-candidate-4-22-dl97gs, error: failed waiting for nodepool=\"np-4gllqb\" for cluster \"cluster-candidate-4-22-4gllqb\" in resourcegroup=\"rg-candidate-4-22-dl97gs\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":4,"runs_affected":2,"failed_runs":2,"run_impact_percent":40,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-05-28..2026-06-03 UTC","prior_windows_present":1,"prior_window_starts":["2026-05-04"],"prior_runs_affected":3,"prior_last_seen_at":"2026-05-09T02:53:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:193]: failed to create node pool \"np-tvvn5b\" for cluster \"cluster-candidate-4-20-tvvn5b\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00102c660\u003e: \n    failed to create NodePool np-tvvn5b, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-tvvn5b in resource group rg-candidate-4-20-xt6lcq, error: failed waiting for nodepool=\"np-tvvn5b\" for cluster \"cluster-candidate-4-20-tvvn5b\" in resourcegroup=\"rg-candidate-4-20-xt6lcq\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:193]: failed to create node pool \"np-tvvn5b\" for cluster \"cluster-candidate-4-20-tvvn5b\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00102c660\u003e: \n    failed to create NodePool np-tvvn5b, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-tvvn5b in resource group rg-candidate-4-20-xt6lcq, error: failed waiting for nodepool=\"np-tvvn5b\" for cluster \"cluster-candidate-4-20-tvvn5b\" in resourcegroup=\"rg-candidate-4-20-xt6lcq\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:193]: failed to create node pool \"np-75wjb5\" for cluster \"cluster-candidate-4-21-75wjb5\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012e46c0\u003e: \n    failed to create NodePool np-75wjb5, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-75wjb5 in resource group rg-candidate-4-21-ppx2fv, error: failed waiting for nodepool=\"np-75wjb5\" for cluster \"cluster-candidate-4-21-75wjb5\" in resourcegroup=\"rg-candidate-4-21-ppx2fv\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:193]: failed to create node pool \"np-75wjb5\" for cluster \"cluster-candidate-4-21-75wjb5\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012e46c0\u003e: \n    failed to create NodePool np-75wjb5, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-75wjb5 in resource group rg-candidate-4-21-ppx2fv, error: failed waiting for nodepool=\"np-75wjb5\" for cluster \"cluster-candidate-4-21-75wjb5\" in resourcegroup=\"rg-candidate-4-21-ppx2fv\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:193]: failed to create node pool \"np-4gllqb\" for cluster \"cluster-candidate-4-22-4gllqb\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015ba570\u003e: \n    failed to create NodePool np-4gllqb, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-4gllqb in resource group rg-candidate-4-22-dl97gs, error: failed waiting for nodepool=\"np-4gllqb\" for cluster \"cluster-candidate-4-22-4gllqb\" in resourcegroup=\"rg-candidate-4-22-dl97gs\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:193]: failed to create node pool \"np-4gllqb\" for cluster \"cluster-candidate-4-22-4gllqb\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015ba570\u003e: \n    failed to create NodePool np-4gllqb, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-4gllqb in resource group rg-candidate-4-22-dl97gs, error: failed waiting for nodepool=\"np-4gllqb\" for cluster \"cluster-candidate-4-22-4gllqb\" in resourcegroup=\"rg-candidate-4-22-dl97gs\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062289277621899264","occurred_at":"2026-06-03T21:44:11Z","signature_id":"dd2bb3a6ce6c60e38bf2afe03c2ebb96c36358e5abbdb2fb36fdcfe197005782","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062239668270796800","occurred_at":"2026-06-03T18:27:06Z","signature_id":"20e64e45c4107d58440e388877b010126d4b87e44ea75bd0c4eec5401719d241","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062239668270796800","occurred_at":"2026-06-03T18:27:06Z","signature_id":"66b72dce0e167200ddcf157740d4fc29f68becf1deafba9539921de2c601b3e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2062239668270796800","occurred_at":"2026-06-03T18:27:06Z","signature_id":"da281ec7ab9e452358dbaa592aecdb0d6e408a4ba60eb6aaeaf14728dec7027e","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}