{"meta":{"start_date":"2026-05-19","end_date":"2026-05-19","start_at":"2026-05-19T00:00:00Z","end_at":"2026-05-20T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T13:11:32Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":2,"failed_runs":1,"raw_occurrences":9,"matched_occurrences":9,"runs_affected":1},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-z2hbpk, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-z2hbpk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-z2hbpk, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-05-13..2026-05-19 UTC","prior_windows_present":4,"prior_window_starts":["2026-04-20","2026-04-27","2026-05-04","2026-05-11"],"prior_runs_affected":28,"prior_last_seen_at":"2026-05-16T00:57:28Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001282b70\u003e: \n    failed to create HCP cluster negative-tests-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-rc85x9, error: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-rc85x9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-rc85x9, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001282b70\u003e: \n    failed to create HCP cluster negative-tests-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-rc85x9, error: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-rc85x9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster negative-tests-cluster in resource group rg-negative-tests-rc85x9, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:109]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000149140\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-rxgmx9, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-candidate-4-22-rxgmx9 in resource group rg-candidate-4-22-2v9rts, error: failed waiting for cluster=\"cluster-candidate-4-22-rxgmx9\" in resourcegroup=\"rg-candidate-4-22-2v9rts\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-candidate-4-22-rxgmx9 in resource group rg-candidate-4-22-2v9rts, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:109]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000149140\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-rxgmx9, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-candidate-4-22-rxgmx9 in resource group rg-candidate-4-22-2v9rts, error: failed waiting for cluster=\"cluster-candidate-4-22-rxgmx9\" in resourcegroup=\"rg-candidate-4-22-2v9rts\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-candidate-4-22-rxgmx9 in resource group rg-candidate-4-22-2v9rts, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001582930\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-z2hbpk, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-z2hbpk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-z2hbpk, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001582930\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-z2hbpk, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-z2hbpk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-z2hbpk, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2056685503272456192","occurred_at":"2026-05-19T10:36:49Z","signature_id":"6535b85f3993181d68560a403ed1d8ae6fccb980d836a15cd4a2a226fcf49932","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2056685503272456192","occurred_at":"2026-05-19T10:36:49Z","signature_id":"664401eae372810191604459c826c2074e0e3533e97a9dd4aec5d279a212c35e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2056685503272456192","occurred_at":"2026-05-19T10:36:49Z","signature_id":"af8fc4fa3438f81440e86f1260fe9cd6a8972cd95cbadd15c7dcf164d46d5da0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2056685503272456192","occurred_at":"2026-05-19T10:36:49Z","signature_id":"eacb4f4d3d23d594a47f59b4be5e5fd84d0c3fe84d2f831644fe52c616db0b6a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"462ac210a3e65f58d7e4fb659028dc9cc1a5acc45dcae793309b9a5143096634","failure_pattern":"failed to cleanup resource group: failed waiting for resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: failed waiting for resourcegroup=\"ephemeral-osdisk-sj9bck\" to finish deleting: context canceled","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-05-13..2026-05-19 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000ebe138\u003e: \n    failed to cleanup resource group: failed waiting for resourcegroup=\"rg-candidate-4-20-7jlf6b\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000ebe138\u003e: \n    failed to cleanup resource group: failed waiting for resourcegroup=\"rg-candidate-4-20-7jlf6b\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001a820c0\u003e: \n    failed to cleanup resource group: failed waiting for resourcegroup=\"ephemeral-osdisk-sj9bck\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001a820c0\u003e: \n    failed to cleanup resource group: failed waiting for resourcegroup=\"ephemeral-osdisk-sj9bck\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0016a0390\u003e: \n    failed to cleanup resource group: failed waiting for resourcegroup=\"np-autoscaling-pq6nsm\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0016a0390\u003e: \n    failed to cleanup resource group: failed waiting for resourcegroup=\"np-autoscaling-pq6nsm\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2056685503272456192","occurred_at":"2026-05-19T10:36:49Z","signature_id":"0772e31535528a2199694e537f779bb284ddf046a5351e9d4a4f1206e7418c41","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2056685503272456192","occurred_at":"2026-05-19T10:36:49Z","signature_id":"3f8cccc3b4198eac61edb1f0adcd67dab1ba7e95ff42bda19c4557a17c36841b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2056685503272456192","occurred_at":"2026-05-19T10:36:49Z","signature_id":"b8f7ba86b464987e52d9d1f35178c2658f689ec0daf8f4f5843e30ae87c0fb45","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1141d0fd9f679007e447ee31146ba219b79d56c1f3a9288dd68187b3f9e7036c","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-c2v6pp\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-c2v6pp, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-05-13..2026-05-19 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00004e510\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-c2v6pp\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-c2v6pp, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00004e510\u003e: \n    failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-c2v6pp\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-c2v6pp, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2056685503272456192","occurred_at":"2026-05-19T10:36:49Z","signature_id":"2c100f1017f2f07836b832a6e5712974dad42d050927c96870dcc71cbc776081","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-2wgl2r\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-2wgl2r, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-05-13..2026-05-19 UTC","prior_windows_present":1,"prior_window_starts":["2026-04-20"],"prior_runs_affected":2,"prior_last_seen_at":"2026-04-24T18:17:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:239]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0007831d0\u003e: \n    failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-2wgl2r\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-2wgl2r, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:239]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0007831d0\u003e: \n    failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-2wgl2r\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-2wgl2r, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2056685503272456192","occurred_at":"2026-05-19T10:36:49Z","signature_id":"65da49a73c114bf4009be516b032aec34514cc294dbdcfee8a2057d5f6c73ce2","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}