{"meta":{"start_date":"2026-05-18","end_date":"2026-05-24","start_at":"2026-05-18T00:00:00Z","end_at":"2026-05-25T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T10:14:30Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":11,"failed_runs":6,"raw_occurrences":16,"matched_occurrences":16,"runs_affected":6},"rows":[{"environment":"int","failure_pattern_id":"e077b5e2625a02cb0ba7300026d32ed84371f6a0e745267fec3071dfdbf320f5","failure_pattern":"Test Panicked","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:410]: Test Panicked","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":7,"runs_affected":2,"failed_runs":2,"run_impact_percent":18.181818181818183,"anchor_occurrences":7,"after_last_push_count":7,"trend_counts":[0,7,0,0,0,0,0,0],"trend_range":"2026-05-17..2026-05-24 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["github.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:410\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0xc000f26640?, {0x7f622ff5a660?, 0xc000f1bef0?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:690 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0xc000eccf00, {0x7f622ff5a660, 0xc000f1bef0}, {0x85b75e7, 0x1b}, {0xc0000140b9, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:404 +0x2a5\ngithub.com/Azure/ARO-HCP/test/e2e.init.func12.1({0x7f622ff5a660, 0xc000f1bef0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:50 +0xe9\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x9102a30?, 0xc000f1bef0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 90\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:410]: Test Panicked: failed to get subscription ID: subscription with name 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' not found\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:410]: Test Panicked\ngithub.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:410\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0xc000f26640?, {0x7f622ff5a660?, 0xc000f1bef0?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:690 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0xc000eccf00, {0x7f622ff5a660, 0xc000f1bef0}, {0x85b75e7, 0x1b}, {0xc0000140b9, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:404 +0x2a5\ngithub.com/Azure/ARO-HCP/test/e2e.init.func12.1({0x7f622ff5a660, 0xc000f1bef0})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:50 +0xe9\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x9102a30?, 0xc000f1bef0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 90\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:410]: Test Panicked: failed to get subscription ID: subscription with name 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' not found\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:410]: Test Panicked","github.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:410\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0xc000dc20c0?, {0x7f02565e3060?, 0xc000bfb200?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:690 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0xc000b30500, {0x7f02565e3060, 0xc000bfb200}, {0x855969a, 0xa}, {0xc0000140b9, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:404 +0x2a5\ngithub.com/Azure/ARO-HCP/test/e2e.init.func15.2.1({0x7f02565e3060, 0xc000bfb200})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:133 +0xe9\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x9102a30?, 0xc000bfb200?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 93\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:410]: Test Panicked: failed to get subscription ID: subscription with name 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' not found\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:410]: Test Panicked\ngithub.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:410\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0xc000dc20c0?, {0x7f02565e3060?, 0xc000bfb200?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:690 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0xc000b30500, {0x7f02565e3060, 0xc000bfb200}, {0x855969a, 0xa}, {0xc0000140b9, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:404 +0x2a5\ngithub.com/Azure/ARO-HCP/test/e2e.init.func15.2.1({0x7f02565e3060, 0xc000bfb200})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:133 +0xe9\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x9102a30?, 0xc000bfb200?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 93\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:410]: Test Panicked: failed to get subscription ID: subscription with name 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' not found\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:410]: Test Panicked","github.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:410\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0xc00151a9b0?, {0x7fae42710d98?, 0xc000734210?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:690 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0xc0005c0c80, {0x7fae42710d98, 0xc000734210}, {0xc001549d60, 0x1a}, {0xc0000140b9, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:404 +0x2a5\ngithub.com/Azure/ARO-HCP/test/e2e.init.func20.1({0x7fae42710d98, 0xc000734210}, {0x853ed2f, 0x4})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:113 +0xcec\nreflect.Value.call({0x749b940?, 0x897e9c0?, 0x13?}, {0x853ee6b, 0x4}, {0xc0007342a0, 0x2, 0x2?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6\nreflect.Value.Call({0x749b940?, 0x897e9c0?, 0x918ce80?}, {0xc0007342a0?, 0x20?, 0x760c9c0?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9\ngithub.com/onsi/ginkgo/v2.invokeFunction({0x749b940, 0x897e9c0}, {0xc000aae220, 0x2, 0x760c9c0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:339 +0x2c5\ngithub.com/onsi/ginkgo/v2.generateTable.func2.1({0x9102a30?, 0xc000734210?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:293 +0x105\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 103\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:410]: Test Panicked: failed to get subscription ID: subscription with name 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' not found\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:410]: Test Panicked\ngithub.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:410\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0xc00151a9b0?, {0x7fae42710d98?, 0xc000734210?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:690 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0xc0005c0c80, {0x7fae42710d98, 0xc000734210}, {0xc001549d60, 0x1a}, {0xc0000140b9, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:404 +0x2a5\ngithub.com/Azure/ARO-HCP/test/e2e.init.func20.1({0x7fae42710d98, 0xc000734210}, {0x853ed2f, 0x4})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:113 +0xcec\nreflect.Value.call({0x749b940?, 0x897e9c0?, 0x13?}, {0x853ee6b, 0x4}, {0xc0007342a0, 0x2, 0x2?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6\nreflect.Value.Call({0x749b940?, 0x897e9c0?, 0x918ce80?}, {0xc0007342a0?, 0x20?, 0x760c9c0?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9\ngithub.com/onsi/ginkgo/v2.invokeFunction({0x749b940, 0x897e9c0}, {0xc000aae220, 0x2, 0x760c9c0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:339 +0x2c5\ngithub.com/onsi/ginkgo/v2.generateTable.func2.1({0x9102a30?, 0xc000734210?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:293 +0x105\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 103\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:410]: Test Panicked: failed to get subscription ID: subscription with name 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' not found\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:410]: Test Panicked"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2056432481736855552","occurred_at":"2026-05-18T17:51:24Z","signature_id":"78c68c5e61789692e543267c4af512813a26836661d654baded2b8baabf587cb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2056432481736855552","occurred_at":"2026-05-18T17:51:24Z","signature_id":"7e7119c6ed64c762d1e91b5981e7b608404315f061bbacb8db66b54851907cda","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2056432481736855552","occurred_at":"2026-05-18T17:51:24Z","signature_id":"a3c7a381f6cc0af88e7875609409b297bfc4a2bd5239db405ac95beb5dac1408","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2056432481736855552","occurred_at":"2026-05-18T17:51:24Z","signature_id":"cd2ddb227dc52098c4ffa773515b6226f5a3c057708872ccb6d832bee324be47","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2056370113375899648","occurred_at":"2026-05-18T13:43:35Z","signature_id":"02fac82c7dd4276c3efe7cc750a2fcc69ed0089be4b7dccaba80b774c6537aaa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2056370113375899648","occurred_at":"2026-05-18T13:43:35Z","signature_id":"4465fe1022fffa342c361ff01420c68834d95d12720a5c0a4a283345970c9a2a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2056370113375899648","occurred_at":"2026-05-18T13:43:35Z","signature_id":"a32ef522f78be573ac6314a61e2c36dfe93c377f0fa1c8f5969625ee3f0455a9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"52d9258f03a60774b49eaa70a514574e25b40d0eeb6ddfa13f252e24164f6be2","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-m8h2c8\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-v2tfvh\" to finish deleting: context canceled","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":4,"runs_affected":2,"failed_runs":2,"run_impact_percent":18.181818181818183,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,4,0,0,0,0,0],"trend_range":"2026-05-17..2026-05-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-05-11"],"prior_runs_affected":10,"prior_last_seen_at":"2026-05-14T21:28:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000940840\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-m8h2c8\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-v2tfvh\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000940840\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-m8h2c8\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-v2tfvh\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0018523d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-6zn88r\" in resourcegroup=\"rg-np-version-upgrade-6zn88r-25m7ll\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0018523d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-6zn88r\" in resourcegroup=\"rg-np-version-upgrade-6zn88r-25m7ll\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013522e8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-rs22qw\" in resourcegroup=\"rg-np-version-upgrade-rs22qw-sssr8k\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013522e8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-rs22qw\" in resourcegroup=\"rg-np-version-upgrade-rs22qw-sssr8k\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2056836814454394880","occurred_at":"2026-05-19T20:38:05Z","signature_id":"31fde93fd86b7a185d10b2360957b602c1b5a79e2c4e4c32df6c2f7fa474a3ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2056836814454394880","occurred_at":"2026-05-19T20:38:05Z","signature_id":"8911a329b249b8cfea37ec895ff435b325a9921c1d65506a0fca9e0875f73a2c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2056836814454394880","occurred_at":"2026-05-19T20:38:05Z","signature_id":"f8910741fe8efe45700aa805d58e52326562d40b03c78b323bc97a0465fb7e7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2056557367306227712","occurred_at":"2026-05-19T02:07:39Z","signature_id":"e4a6b69af206fc19abbdeec381abbd36ed86b3d36b118d2897cf316a2459ad29","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"98e32667c3e390e96a18d1f5c467888c22b014b872e3aaaa2bfc4f15efac6c45","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":18.181818181818183,"anchor_occurrences":2,"after_last_push_count":6,"trend_counts":[0,1,1,0,0,0,0,0],"trend_range":"2026-05-17..2026-05-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-04-20","2026-04-27","2026-05-04","2026-05-11"],"prior_runs_affected":7,"prior_last_seen_at":"2026-05-16T13:08:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00084c2c0\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-hhw7f.apps.aro.g4j7l0c1u5w2t3k.5arn.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00084c2c0\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-hhw7f.apps.aro.g4j7l0c1u5w2t3k.5arn.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000f227a0\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-zsz8g.apps.aro.d4g2w2k1g5y5i9u.77mw.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000f227a0\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-zsz8g.apps.aro.d4g2w2k1g5y5i9u.77mw.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2056624196653420544","occurred_at":"2026-05-19T06:33:13Z","signature_id":"93c3728651168bdecdf404c8b527adf12c701b32704fec3e8d2cf7153611230e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2056510773210910720","occurred_at":"2026-05-18T23:02:31Z","signature_id":"2509fe4213faf0ba0ef946240cf31cecdddf3a5f9494845e3569356596e2cec5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"0ece011306dab74cd22dd19f1f3c89bbdb930a2321b61be59e4a6a90aae6e3eb","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-67kmqk in resource group rg-zstream-upgrade-4-19-8lwkkf, error: failed waiting for cluster=\"cluster-zstream-4-19-67kmqk\" in resourcegroup=\"rg-zstream-upgrade-4-19-8lwkkf\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-67kmqk in resource group rg-zstream-upgrade-4-19-8lwkkf, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-05-17..2026-05-24 UTC","prior_windows_present":4,"prior_window_starts":["2026-04-20","2026-04-27","2026-05-04","2026-05-11"],"prior_runs_affected":22,"prior_last_seen_at":"2026-05-14T21:28:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:131]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009aa570\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-9vrtlt, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-9vrtlt in resource group rg-np-version-upgrade-9vrtlt-7sfdg6, error: failed waiting for cluster=\"np-version-upgrade-cluster-9vrtlt\" in resourcegroup=\"rg-np-version-upgrade-9vrtlt-7sfdg6\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-9vrtlt in resource group rg-np-version-upgrade-9vrtlt-7sfdg6, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:131]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009aa570\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-9vrtlt, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-9vrtlt in resource group rg-np-version-upgrade-9vrtlt-7sfdg6, error: failed waiting for cluster=\"np-version-upgrade-cluster-9vrtlt\" in resourcegroup=\"rg-np-version-upgrade-9vrtlt-7sfdg6\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-version-upgrade-cluster-9vrtlt in resource group rg-np-version-upgrade-9vrtlt-7sfdg6, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001713020\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-67kmqk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-67kmqk in resource group rg-zstream-upgrade-4-19-8lwkkf, error: failed waiting for cluster=\"cluster-zstream-4-19-67kmqk\" in resourcegroup=\"rg-zstream-upgrade-4-19-8lwkkf\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-67kmqk in resource group rg-zstream-upgrade-4-19-8lwkkf, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001713020\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-67kmqk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-67kmqk in resource group rg-zstream-upgrade-4-19-8lwkkf, error: failed waiting for cluster=\"cluster-zstream-4-19-67kmqk\" in resourcegroup=\"rg-zstream-upgrade-4-19-8lwkkf\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-67kmqk in resource group rg-zstream-upgrade-4-19-8lwkkf, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2056557367306227712","occurred_at":"2026-05-19T02:07:39Z","signature_id":"170a167eb22248e1639caef34aa49fa003a3bb61081d6c0218f1398090d3c611","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2056557367306227712","occurred_at":"2026-05-19T02:07:39Z","signature_id":"75e3cf0ee2fc44545e383e59530a0a680ff7c31dbc7595c3ca4e601b1520dd49","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"88fb289639ac585d41e102922a6ea4f479e947e9962ca681abe481db2908b42c","failure_pattern":"context deadline exceeded","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":1,"after_last_push_count":8,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-05-17..2026-05-24 UTC","prior_windows_present":3,"prior_window_starts":["2026-04-20","2026-05-04","2026-05-11"],"prior_runs_affected":10,"prior_last_seen_at":"2026-05-13T23:17:23Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:132]: Unexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:132]: Unexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2056836814454394880","occurred_at":"2026-05-19T20:38:05Z","signature_id":"b15c55d32190fb3d32f16f58a52f080055a52cddf394a6cd7e062c8467f788e4","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}