{"meta":{"start_date":"2026-06-11","end_date":"2026-06-11","start_at":"2026-06-11T00:00:00Z","end_at":"2026-06-12T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T11:05:55Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":5,"failed_runs":5,"raw_occurrences":17,"matched_occurrences":17,"runs_affected":5},"rows":[{"environment":"int","failure_pattern_id":"a880b7dbbe8686372b606e0212fd3742bc75c3c0a9c545bae9481b243ed0f55b","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-7cb2r8 in resource group rg-candidate-4-20-mbsl7c, error: failed waiting for nodepool=\"np-7cb2r8\" for cluster \"cluster-candidate-4-20-7cb2r8\" in resourcegroup=\"rg-candidate-4-20-mbsl7c\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":7,"runs_affected":4,"failed_runs":4,"run_impact_percent":80,"anchor_occurrences":7,"after_last_push_count":7,"trend_counts":[0,0,0,0,0,0,7],"trend_range":"2026-06-05..2026-06-11 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-01"],"prior_runs_affected":15,"prior_last_seen_at":"2026-06-04T21:46:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-vfp6gc\" for cluster \"cluster-candidate-4-22-vfp6gc\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00111b320\u003e: \n    failed to create NodePool np-vfp6gc, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-vfp6gc in resource group rg-candidate-4-22-k7jr2t, error: failed waiting for nodepool=\"np-vfp6gc\" for cluster \"cluster-candidate-4-22-vfp6gc\" in resourcegroup=\"rg-candidate-4-22-k7jr2t\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-vfp6gc\" for cluster \"cluster-candidate-4-22-vfp6gc\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00111b320\u003e: \n    failed to create NodePool np-vfp6gc, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-vfp6gc in resource group rg-candidate-4-22-k7jr2t, error: failed waiting for nodepool=\"np-vfp6gc\" for cluster \"cluster-candidate-4-22-vfp6gc\" in resourcegroup=\"rg-candidate-4-22-k7jr2t\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-xxwcnw\" for cluster \"cluster-candidate-4-21-xxwcnw\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000c0d7a0\u003e: \n    failed to create NodePool np-xxwcnw, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-xxwcnw in resource group rg-candidate-4-21-bm2dvr, error: failed waiting for nodepool=\"np-xxwcnw\" for cluster \"cluster-candidate-4-21-xxwcnw\" in resourcegroup=\"rg-candidate-4-21-bm2dvr\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-xxwcnw\" for cluster \"cluster-candidate-4-21-xxwcnw\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000c0d7a0\u003e: \n    failed to create NodePool np-xxwcnw, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-xxwcnw in resource group rg-candidate-4-21-bm2dvr, error: failed waiting for nodepool=\"np-xxwcnw\" for cluster \"cluster-candidate-4-21-xxwcnw\" in resourcegroup=\"rg-candidate-4-21-bm2dvr\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-bc29t2\" for cluster \"cluster-candidate-4-21-bc29t2\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001422bd0\u003e: \n    failed to create NodePool np-bc29t2, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-bc29t2 in resource group rg-candidate-4-21-zckkfx, error: failed waiting for nodepool=\"np-bc29t2\" for cluster \"cluster-candidate-4-21-bc29t2\" in resourcegroup=\"rg-candidate-4-21-zckkfx\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-bc29t2\" for cluster \"cluster-candidate-4-21-bc29t2\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001422bd0\u003e: \n    failed to create NodePool np-bc29t2, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-bc29t2 in resource group rg-candidate-4-21-zckkfx, error: failed waiting for nodepool=\"np-bc29t2\" for cluster \"cluster-candidate-4-21-bc29t2\" in resourcegroup=\"rg-candidate-4-21-zckkfx\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2065210211181793280","occurred_at":"2026-06-11T23:10:58Z","signature_id":"7e00f5ea89a668f7e39b394b837e5758b99143c17d9086036cc32ad0d052175f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2065062586193285120","occurred_at":"2026-06-11T13:23:31Z","signature_id":"15aaca0a3800ea3b625252b9b43214d83f39156a304160148cd0026ddde992b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2064941763205795840","occurred_at":"2026-06-11T05:24:15Z","signature_id":"e4576a456b6ea048e0bda32827977ce8cdd3697a5e8b54648d8045c43a5d32a9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2064941763205795840","occurred_at":"2026-06-11T05:24:15Z","signature_id":"ed395ec2a95d820b6780d5a29a3e95a78115d5120600fdc7ebc1bb397b83518f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2064941763205795840","occurred_at":"2026-06-11T05:24:15Z","signature_id":"f2a1306ab5fe414f5c50c512631a0bf34577499a83f801207709fe08af9e0fdd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2064866707712774144","occurred_at":"2026-06-11T00:26:01Z","signature_id":"98ae70aedba81d474350d36027dad5a9ae7e436fe97c2fbf00b2931252bfe3fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2064866707712774144","occurred_at":"2026-06-11T00:26:01Z","signature_id":"f5bb03b280d00a5f6b91a7e7a16f9686cc61c4247b64e05a042de850e0b4b3a3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"0ece011306dab74cd22dd19f1f3c89bbdb930a2321b61be59e4a6a90aae6e3eb","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-nightly-4-23-7lkzr7\" in resourcegroup=\"rg-nightly-4-23-5p5rzq\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-7lkzr7 in resource group rg-nightly-4-23-5p5rzq, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":5,"runs_affected":3,"failed_runs":3,"run_impact_percent":60,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,0,5],"trend_range":"2026-06-05..2026-06-11 UTC","prior_windows_present":4,"prior_window_starts":["2026-05-11","2026-05-18","2026-05-25","2026-06-01"],"prior_runs_affected":31,"prior_last_seen_at":"2026-06-05T18:49:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-wwmkpj/cluster-nightly-5-0-vh6qwq should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001262480\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-vh6qwq\" in resourcegroup=\"rg-nightly-5-0-wwmkpj\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-vh6qwq in resource group rg-nightly-5-0-wwmkpj, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-wwmkpj/cluster-nightly-5-0-vh6qwq should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001262480\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-vh6qwq\" in resourcegroup=\"rg-nightly-5-0-wwmkpj\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-vh6qwq in resource group rg-nightly-5-0-wwmkpj, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-jvdz4s/cluster-nightly-4-23-h8bmck should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012ef830\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-h8bmck\" in resourcegroup=\"rg-nightly-4-23-jvdz4s\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-h8bmck in resource group rg-nightly-4-23-jvdz4s, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-jvdz4s/cluster-nightly-4-23-h8bmck should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012ef830\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-h8bmck\" in resourcegroup=\"rg-nightly-4-23-jvdz4s\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-h8bmck in resource group rg-nightly-4-23-jvdz4s, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-tgndsp/cluster-nightly-4-23-sxpl48 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011aa810\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-sxpl48\" in resourcegroup=\"rg-nightly-4-23-tgndsp\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-sxpl48 in resource group rg-nightly-4-23-tgndsp, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-tgndsp/cluster-nightly-4-23-sxpl48 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011aa810\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-sxpl48\" in resourcegroup=\"rg-nightly-4-23-tgndsp\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-sxpl48 in resource group rg-nightly-4-23-tgndsp, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2065210211181793280","occurred_at":"2026-06-11T23:10:58Z","signature_id":"097ae6e90370bdd4f9c78a6e8dabf6754d53196682cf1d47bf62872001e43742","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2065210211181793280","occurred_at":"2026-06-11T23:10:58Z","signature_id":"f472177c85e442eb4ac1686400ff8a54af50ecc718fb147d08d707394ea7fdce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2065062586193285120","occurred_at":"2026-06-11T13:23:31Z","signature_id":"76c9b31f9fbe7be8875d64ffe507f0624ac1f87160ef91f97df254025147b966","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2064866707712774144","occurred_at":"2026-06-11T00:26:01Z","signature_id":"740aad251ad3711069c4489005a624de31455a966e0c1e440b5db633c163d681","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2064866707712774144","occurred_at":"2026-06-11T00:26:01Z","signature_id":"7d247648af00b1f9e54b68e838c574c7e231ce3e5c84f5c8a7c7600cd0f9bb77","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"52d9258f03a60774b49eaa70a514574e25b40d0eeb6ddfa13f252e24164f6be2","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-mdzh4m\" to finish deleting: context canceled","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-06-05..2026-06-11 UTC","prior_windows_present":4,"prior_window_starts":["2026-05-11","2026-05-18","2026-05-25","2026-06-01"],"prior_runs_affected":14,"prior_last_seen_at":"2026-06-05T07:18:21Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should update NodeDrainTimeoutMinutes and upgrade a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:282] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000d36660\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-mdzh4m\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:282] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000d36660\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-mdzh4m\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:282] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000caa390\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-drain-timeout-and-upgrade-nk7mdh\" in resourcegroup=\"rg-np-drain-timeout-and-upgrade-nk7mdh-z5twvm\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:282] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000caa390\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-drain-timeout-and-upgrade-nk7mdh\" in resourcegroup=\"rg-np-drain-timeout-and-upgrade-nk7mdh-z5twvm\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:282] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001678198\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-7ltfqs\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:282] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001678198\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-7ltfqs\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2064866707712774144","occurred_at":"2026-06-11T00:26:01Z","signature_id":"77d4f705efa3cfb7026fee97a24ddd0a96dd45e80b4ff1a0c5d1f0c8403df115","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2064866707712774144","occurred_at":"2026-06-11T00:26:01Z","signature_id":"ebf4c915d9f1bd2dc2c40fd448de3b8cd3dbc867572584bf149a7bcee2433f23","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2064866707712774144","occurred_at":"2026-06-11T00:26:01Z","signature_id":"ffd54b57fd3f9302a0ab9065530866e414f77a42b40b9ff9fa78cea0307af9ef","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"6bd47b9adbcb9d9a7acc8d32204c9d514430e524b2fdab25834d7d2c3a57f4b6","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message no upgrade path found from 4.21.15 to 4.22.0: no reachable versions in target minor and no gateway version in current minor; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-06-05..2026-06-11 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:215]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-t6zgnn\" to 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000daa9a0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-t6zgnn\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-65ltw8\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2b654c62-7bae-4f51-828c-b10da7a88688\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2b654c62-7bae-4f51-828c-b10da7a88688\",\n      \"name\": \"2b654c62-7bae-4f51-828c-b10da7a88688\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-06-11T20:21:45.858985705Z\",\n      \"endTime\": \"2026-06-11T20:22:38.89726274Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"no upgrade path found from 4.21.15 to 4.22.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:215]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-t6zgnn\" to 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000daa9a0\u003e: \n    failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-t6zgnn\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-65ltw8\" to finish updating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2b654c62-7bae-4f51-828c-b10da7a88688\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2b654c62-7bae-4f51-828c-b10da7a88688\",\n      \"name\": \"2b654c62-7bae-4f51-828c-b10da7a88688\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-06-11T20:21:45.858985705Z\",\n      \"endTime\": \"2026-06-11T20:22:38.89726274Z\",\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"no upgrade path found from 4.21.15 to 4.22.0: no reachable versions in target minor and no gateway version in current minor\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2065155767597535232","occurred_at":"2026-06-11T19:34:38Z","signature_id":"b15151dcf083c8226670dc929f7709ea569a75419d50a5c2be098e477b5ca63e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"e457d9ded8188e2e5d3bf94632039456ab0d3f40aab17c66a163827314f30476","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-06-05..2026-06-11 UTC","prior_windows_present":1,"prior_window_starts":["2026-05-25"],"prior_runs_affected":2,"prior_last_seen_at":"2026-05-27T09:38:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001081098\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-b248kl, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-b248kl\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001081098\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-b248kl, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-b248kl\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2064941763205795840","occurred_at":"2026-06-11T05:24:15Z","signature_id":"a5754982c1c05a8010e80f1238eea03577248a44c458d8047c79b6ecc5e7b9ff","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}