{"meta":{"start_date":"2026-08-01","end_date":"2026-08-01","start_at":"2026-08-01T00:00:00Z","end_at":"2026-08-02T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T04:03:06Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":6,"failed_runs":3,"raw_occurrences":7,"matched_occurrences":7,"runs_affected":3},"rows":[{"environment":"int","failure_pattern_id":"f90cae3228fe72b2427515e7c51834eebdc3e73c0b1aa6bca2abedd53a2e7578","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"admin-credential-lifecycle-test-f5xlqsfqjmgk\", caused by: failed waiting for hcpCluster=\"admin-cred-lifecycle-77tq44\" in resourcegroup=\"admin-credential-lifecycle-test-f5xlqsfqjmgk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-77tq44 in resource group admin-credential-lifecycle-test-f5xlqsfqjmgk, error: context deadline exceeded, error: failed waiting for hcpCluster=\"admin-cred-lifecycle-77tq44\" in resourcegroup=\"admin-credential-lifecycle-test-f5xlqsfqjmgk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-77tq44 in resource group admin-credential-lifecycle-test-f5xlqsfqjmgk, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":7,"runs_affected":3,"failed_runs":3,"run_impact_percent":50,"anchor_occurrences":7,"after_last_push_count":21,"trend_counts":[0,2,1,2,1,1,7],"trend_range":"2026-07-26..2026-08-01 UTC","prior_windows_present":3,"prior_window_starts":["2026-07-06","2026-07-13","2026-07-20"],"prior_runs_affected":16,"prior_last_seen_at":"2026-07-22T16:58:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000db2990\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"admin-credential-lifecycle-test-f5xlqsfqjmgk\", caused by: failed waiting for hcpCluster=\"admin-cred-lifecycle-77tq44\" in resourcegroup=\"admin-credential-lifecycle-test-f5xlqsfqjmgk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-77tq44 in resource group admin-credential-lifecycle-test-f5xlqsfqjmgk, error: context deadline exceeded, error: failed waiting for hcpCluster=\"admin-cred-lifecycle-77tq44\" in resourcegroup=\"admin-credential-lifecycle-test-f5xlqsfqjmgk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-77tq44 in resource group admin-credential-lifecycle-test-f5xlqsfqjmgk, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc000db2990\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"admin-credential-lifecycle-test-f5xlqsfqjmgk\", caused by: failed waiting for hcpCluster=\"admin-cred-lifecycle-77tq44\" in resourcegroup=\"admin-credential-lifecycle-test-f5xlqsfqjmgk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-77tq44 in resource group admin-credential-lifecycle-test-f5xlqsfqjmgk, error: context deadline exceeded, error: failed waiting for hcpCluster=\"admin-cred-lifecycle-77tq44\" in resourcegroup=\"admin-credential-lifecycle-test-f5xlqsfqjmgk\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster admin-cred-lifecycle-77tq44 in resource group admin-credential-lifecycle-test-f5xlqsfqjmgk, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001076b10\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-noedge-vlq6gm-8md5vxnw7l5r\", caused by: failed waiting for hcpCluster=\"np-noedge-vlq6gm\" in resourcegroup=\"rg-np-noedge-vlq6gm-8md5vxnw7l5r\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-noedge-vlq6gm in resource group rg-np-noedge-vlq6gm-8md5vxnw7l5r, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-noedge-vlq6gm\" in resourcegroup=\"rg-np-noedge-vlq6gm-8md5vxnw7l5r\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-noedge-vlq6gm in resource group rg-np-noedge-vlq6gm-8md5vxnw7l5r, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc001076b10\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-np-noedge-vlq6gm-8md5vxnw7l5r\", caused by: failed waiting for hcpCluster=\"np-noedge-vlq6gm\" in resourcegroup=\"rg-np-noedge-vlq6gm-8md5vxnw7l5r\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-noedge-vlq6gm in resource group rg-np-noedge-vlq6gm-8md5vxnw7l5r, error: context deadline exceeded, error: failed waiting for hcpCluster=\"np-noedge-vlq6gm\" in resourcegroup=\"rg-np-noedge-vlq6gm-8md5vxnw7l5r\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster np-noedge-vlq6gm in resource group rg-np-noedge-vlq6gm-8md5vxnw7l5r, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00129c330\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"external-auth-cluster-ww59wv7wsf5c\", caused by: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-ww59wv7wsf5c\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ea-cluster in resource group external-auth-cluster-ww59wv7wsf5c, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-ww59wv7wsf5c\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ea-cluster in resource group external-auth-cluster-ww59wv7wsf5c, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00129c330\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"external-auth-cluster-ww59wv7wsf5c\", caused by: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-ww59wv7wsf5c\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ea-cluster in resource group external-auth-cluster-ww59wv7wsf5c, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-ww59wv7wsf5c\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ea-cluster in resource group external-auth-cluster-ww59wv7wsf5c, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083661568511315968","occurred_at":"2026-08-01T21:10:05Z","signature_id":"0f904af658625547f2625ae558f4cb3550bcbe5b8b1552664be79ba8a249aaea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083661568511315968","occurred_at":"2026-08-01T21:10:05Z","signature_id":"7b47052217e18790be4f3e94628783ce9103c711570915db12f5cc52c398bf02","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083661568511315968","occurred_at":"2026-08-01T21:10:05Z","signature_id":"a59aab1068cc39f99f03e7d2d1f00af73481b9177294c34a56bc78879c5d6982","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083661568511315968","occurred_at":"2026-08-01T21:10:05Z","signature_id":"c26c51ac2e5a43f00675f492dae1a39c5790b351afd23da852546c9b229b60c0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083458381179260928","occurred_at":"2026-08-01T07:42:41Z","signature_id":"8546d745faaec6484b923fbde7c857adfdb77f38724b3bba967d7e8c6b1e508f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083366532309585920","occurred_at":"2026-08-01T01:37:43Z","signature_id":"40f35ad9ee71970ee1addeb8b956b42dfe6f8a7469513fecfde1464979464e8f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2083366532309585920","occurred_at":"2026-08-01T01:37:43Z","signature_id":"ff3410283562ab455f05b6661eec03613dc61a18f57f4cdf6461682f200f2071","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}