{"meta":{"start_date":"2026-05-20","end_date":"2026-05-20","start_at":"2026-05-20T00:00:00Z","end_at":"2026-05-21T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T13:52:21Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":8,"failed_runs":3,"raw_occurrences":12,"matched_occurrences":12,"runs_affected":3},"rows":[{"environment":"prod","failure_pattern_id":"f27b30bef0ebb941cd79b083d3eb5ff3005b4b85f92582e4349d8748b1eaeb94","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-7tpw8l\" to finish deleting: context canceled","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":5,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,0,5],"trend_range":"2026-05-14..2026-05-20 UTC","prior_windows_present":2,"prior_window_starts":["2026-04-20","2026-05-04"],"prior_runs_affected":3,"prior_last_seen_at":"2026-05-07T21:40:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000b9c438\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-9v22th\" in resourcegroup=\"rg-np-version-upgrade-9v22th-5s2l57\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000b9c438\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-9v22th\" in resourcegroup=\"rg-np-version-upgrade-9v22th-5s2l57\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00145c258\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-5lhhw2\" in resourcegroup=\"rg-np-version-upgrade-5lhhw2-s7gzkl\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00145c258\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-5lhhw2\" in resourcegroup=\"rg-np-version-upgrade-5lhhw2-s7gzkl\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000ca6168\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-7tpw8l\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:262] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000ca6168\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-7tpw8l\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2057236919435137024","occurred_at":"2026-05-20T23:07:55Z","signature_id":"1a29075d462120df05b781a534723d404596354493e385f2b1dffba8eade58fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2057236919435137024","occurred_at":"2026-05-20T23:07:55Z","signature_id":"1f3c5c1f7ae4064bfca82ce59353e531b9fb18dde34691a5db2bdf1068c5804b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2057236919435137024","occurred_at":"2026-05-20T23:07:55Z","signature_id":"547514199866042c5d5c60a144a5db5e92896578b42c035630706671ea26db84","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2057236919435137024","occurred_at":"2026-05-20T23:07:55Z","signature_id":"aef680185aedd693d6ef151a4ef9061231179a33e154002dd0c7ecfc1def9357","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2057236919435137024","occurred_at":"2026-05-20T23:07:55Z","signature_id":"f8ec1016106580644716e638111bba4fa9c015cea0ee390136209101df7afd33","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":2,"after_last_push_count":19,"trend_counts":[3,0,0,0,0,0,2],"trend_range":"2026-05-14..2026-05-20 UTC","prior_windows_present":4,"prior_window_starts":["2026-04-20","2026-04-27","2026-05-04","2026-05-11"],"prior_runs_affected":9,"prior_last_seen_at":"2026-05-14T02:42:27Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["od/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:199\n          \u003e github.com/onsi/ginkgo/v2/internal.(*group).run(0xc000de0b88, {0xc000d1f1e0?, 0x0?, 0x0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:349\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).runSpecs(0xc000da0008, {0x0, 0x0}, {0xc6e2ca0, 0x0, 0x0}, {0xc00007a094, 0x2a}, 0x0, {0xc000d1f1a0, ...})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:496\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).RunSpec(_, {_, _}, {_, _, _}, {_, _}, {0xc00007a094, 0x2a}, ...)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite_patch.go:68\n            github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite.func1.1({0x2e18a85?, 0xc000c01ec0?})\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260422120742-50dec0b5ecf3/pkg/ginkgo/util.go:84\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.runSpec({0x90e87c8, 0xc0008b5590}, 0xc000e8b520, 0x1)\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260422120742-50dec0b5ecf3/pkg/extension/extensiontests/spec.go:596\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260422120742-50dec0b5ecf3/pkg/extension/extensiontests/spec.go:237\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260422120742-50dec0b5ecf3/pkg/extension/extensiontests/spec.go:230\n\n          goroutine 107 [select]\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2(0xc000e8b520?)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:131\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts in goroutine 106\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:128\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:119 @ 05/21/26 01:52:31.225\n  \"ts\"=\"2026-05-21 01:52:31.225980\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-05-21 01:52:31.226030\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-candidate-4-21-6spgxp-87qlp2\" \"resourceGroup\"=\"rg-candidate-4-21-rvqdlq\"\n  \"ts\"=\"2026-05-21 01:52:31.949212\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-candidate-4-21-6spgxp-87qlp2\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-05-21 01:52:33.901519\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-6427b6424c4038306dc527fd5f7fc0b377fe9c26e56432adfe480dff6189e0c7.yaml.gz\"\n  \"ts\"=\"2026-05-21 01:52:33.901684\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-6427b6424c4038306dc527fd5f7fc0b377fe9c26e56432adfe480dff6189e0c7.yaml.gz\"\n  \"ts\"=\"2026-05-21 01:52:33.901758\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-05-21 01:52:33.903207\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-67\"\n  \"ts\"=\"2026-05-21 01:52:33.903226\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-candidate-4-21-rvqdlq\"\n  \"ts\"=\"2026-05-21 01:52:34.245138\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-67\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-05-21 01:52:34.297415\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-candidate-4-21-rvqdlq\" \"deployment\"=\"pooledNonMsiScopedAssignments-txlkb6a3h2s3w\"\n  \"ts\"=\"2026-05-21 01:52:34.355759\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-67\" \"deployment\"=\"msi-deployment-67\"\n  \"ts\"=\"2026-05-21 01:52:34.536073\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-candidate-4-21-rvqdlq\" \"deployment\"=\"customer-infra-cluster-candidate-4-21-6spgxp-87qlp2\"\n  \"ts\"=\"2026-05-21 01:52:34.830797\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-05-21 01:52:35.321929\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-05-21 01:52:35.322167\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-candidate-4-21-rvqdlq\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:199\n          \u003e github.com/onsi/ginkgo/v2/internal.(*group).run(0xc000de0b88, {0xc000d1f1e0?, 0x0?, 0x0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:349\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).runSpecs(0xc000da0008, {0x0, 0x0}, {0xc6e2ca0, 0x0, 0x0}, {0xc00007a094, 0x2a}, 0x0, {0xc000d1f1a0, ...})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:496\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).RunSpec(_, {_, _}, {_, _, _}, {_, _}, {0xc00007a094, 0x2a}, ...)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite_patch.go:68\n            github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite.func1.1({0x2e18a85?, 0xc000c01ec0?})\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260422120742-50dec0b5ecf3/pkg/ginkgo/util.go:84\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.runSpec({0x90e87c8, 0xc0008b5590}, 0xc000e8b520, 0x1)\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260422120742-50dec0b5ecf3/pkg/extension/extensiontests/spec.go:596\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260422120742-50dec0b5ecf3/pkg/extension/extensiontests/spec.go:237\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260422120742-50dec0b5ecf3/pkg/extension/extensiontests/spec.go:230\n\n          goroutine 107 [select]\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2(0xc000e8b520?)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:131\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts in goroutine 106\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:128\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:119 @ 05/21/26 01:52:31.225\n  \"ts\"=\"2026-05-21 01:52:31.225980\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-05-21 01:52:31.226030\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-candidate-4-21-6spgxp-87qlp2\" \"resourceGroup\"=\"rg-candidate-4-21-rvqdlq\"\n  \"ts\"=\"2026-05-21 01:52:31.949212\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-candidate-4-21-6spgxp-87qlp2\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-05-21 01:52:33.901519\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-6427b6424c4038306dc527fd5f7fc0b377fe9c26e56432adfe480dff6189e0c7.yaml.gz\"\n  \"ts\"=\"2026-05-21 01:52:33.901684\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-6427b6424c4038306dc527fd5f7fc0b377fe9c26e56432adfe480dff6189e0c7.yaml.gz\"\n  \"ts\"=\"2026-05-21 01:52:33.901758\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-05-21 01:52:33.903207\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-67\"\n  \"ts\"=\"2026-05-21 01:52:33.903226\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-candidate-4-21-rvqdlq\"\n  \"ts\"=\"2026-05-21 01:52:34.245138\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-67\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-05-21 01:52:34.297415\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-candidate-4-21-rvqdlq\" \"deployment\"=\"pooledNonMsiScopedAssignments-txlkb6a3h2s3w\"\n  \"ts\"=\"2026-05-21 01:52:34.355759\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-67\" \"deployment\"=\"msi-deployment-67\"\n  \"ts\"=\"2026-05-21 01:52:34.536073\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-candidate-4-21-rvqdlq\" \"deployment\"=\"customer-infra-cluster-candidate-4-21-6spgxp-87qlp2\"\n  \"ts\"=\"2026-05-21 01:52:34.830797\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-05-21 01:52:35.321929\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-05-21 01:52:35.322167\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-candidate-4-21-rvqdlq\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","ing specs.  Interrupt again to skip cleanup.\n    Here's a current progress report:\n      Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request (Spec Runtime: 2h29m59.885s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:44\n        In [It] (Node Runtime: 2h29m59.885s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:44\n          At [By Step] creating the HCP cluster (Step Runtime: 31m42.234s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:76\n\n          Spec Goroutine\n          goroutine 116 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x90e87c8, 0xc000daac30}, 0x85624cf?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x91280a0, {0x90e87c8, 0xc000daac30}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x90e8838?, 0xc000b193b0?}, {{0x9102420?, 0xc00100a280?}, 0xc001989050?}, 0xc00008f750, {0xc000966420, 0x11}, {0x85827de, 0x12}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:850\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0xc000b009e7, 0x7}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:346\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func15.1.1({0x7f6e2c6e7860, 0xc001016810})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:77\n                | \n                | By(\"creating the HCP cluster\")\n                \u003e err = tc.CreateHCPClusterFromParam(\n                | \tctx,\n                | \tGinkgoLogr,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x9102a30?, 0xc001016810?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 114\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84 @ 05/21/26 01:52:31.235\n  \"ts\"=\"2026-05-21 01:52:31.236130\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-05-21 01:52:31.236193\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-patch-name-cluster-s9mrsr\" \"resourceGroup\"=\"patch-name-6glw6m\"\n  \"ts\"=\"2026-05-21 01:52:31.505346\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-patch-name-cluster-s9mrsr\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-05-21 01:52:32.976416\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-edcc54feb78eb6f23e76e41140aef2c970127915770a00eb26db2362637d7984.yaml.gz\"\n  \"ts\"=\"2026-05-21 01:52:32.976513\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-edcc54feb78eb6f23e76e41140aef2c970127915770a00eb26db2362637d7984.yaml.gz\"\n  \"ts\"=\"2026-05-21 01:52:32.976597\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-05-21 01:52:32.978037\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-40\"\n  \"ts\"=\"2026-05-21 01:52:32.978051\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"patch-name-6glw6m\"\n  \"ts\"=\"2026-05-21 01:52:33.371789\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-40\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-05-21 01:52:33.434158\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"patch-name-6glw6m\" \"deployment\"=\"pooledNonMsiScopedAssignments-rr7tdjqoygobu\"\n  \"ts\"=\"2026-05-21 01:52:33.487353\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-40\" \"deployment\"=\"msi-deployment-40\"\n  \"ts\"=\"2026-05-21 01:52:33.673654\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"patch-name-6glw6m\" \"deployment\"=\"customer-infra-patch-name-cluster-s9mrsr\"\n  \"ts\"=\"2026-05-21 01:52:33.945032\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-05-21 01:52:34.176123\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-05-21 01:52:34.176366\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"patch-name-6glw6m\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\ning specs.  Interrupt again to skip cleanup.\n    Here's a current progress report:\n      Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request (Spec Runtime: 2h29m59.885s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:44\n        In [It] (Node Runtime: 2h29m59.885s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:44\n          At [By Step] creating the HCP cluster (Step Runtime: 31m42.234s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:76\n\n          Spec Goroutine\n          goroutine 116 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x90e87c8, 0xc000daac30}, 0x85624cf?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x91280a0, {0x90e87c8, 0xc000daac30}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x90e8838?, 0xc000b193b0?}, {{0x9102420?, 0xc00100a280?}, 0xc001989050?}, 0xc00008f750, {0xc000966420, 0x11}, {0x85827de, 0x12}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:850\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0xc000b009e7, 0x7}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:346\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func15.1.1({0x7f6e2c6e7860, 0xc001016810})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:77\n                | \n                | By(\"creating the HCP cluster\")\n                \u003e err = tc.CreateHCPClusterFromParam(\n                | \tctx,\n                | \tGinkgoLogr,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x9102a30?, 0xc001016810?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 114\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84 @ 05/21/26 01:52:31.235\n  \"ts\"=\"2026-05-21 01:52:31.236130\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-05-21 01:52:31.236193\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-patch-name-cluster-s9mrsr\" \"resourceGroup\"=\"patch-name-6glw6m\"\n  \"ts\"=\"2026-05-21 01:52:31.505346\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-patch-name-cluster-s9mrsr\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-05-21 01:52:32.976416\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-edcc54feb78eb6f23e76e41140aef2c970127915770a00eb26db2362637d7984.yaml.gz\"\n  \"ts\"=\"2026-05-21 01:52:32.976513\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-edcc54feb78eb6f23e76e41140aef2c970127915770a00eb26db2362637d7984.yaml.gz\"\n  \"ts\"=\"2026-05-21 01:52:32.976597\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-05-21 01:52:32.978037\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-40\"\n  \"ts\"=\"2026-05-21 01:52:32.978051\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"patch-name-6glw6m\"\n  \"ts\"=\"2026-05-21 01:52:33.371789\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-40\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-05-21 01:52:33.434158\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"patch-name-6glw6m\" \"deployment\"=\"pooledNonMsiScopedAssignments-rr7tdjqoygobu\"\n  \"ts\"=\"2026-05-21 01:52:33.487353\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-40\" \"deployment\"=\"msi-deployment-40\"\n  \"ts\"=\"2026-05-21 01:52:33.673654\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"patch-name-6glw6m\" \"deployment\"=\"customer-infra-patch-name-cluster-s9mrsr\"\n  \"ts\"=\"2026-05-21 01:52:33.945032\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-05-21 01:52:34.176123\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-05-21 01:52:34.176366\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"patch-name-6glw6m\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2057236919435137024","occurred_at":"2026-05-20T23:07:55Z","signature_id":"04db2ea8194d14c854164b691230af34c1b193d2ef8a1609f9d13f66cc475de5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2057236919435137024","occurred_at":"2026-05-20T23:07:55Z","signature_id":"7db201aca0e4fd538915530fa855d8602b087e3bfb702d0d57f38811eb830ff7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-05-14..2026-05-20 UTC","prior_windows_present":3,"prior_window_starts":["2026-04-20","2026-04-27","2026-05-04"],"prior_runs_affected":6,"prior_last_seen_at":"2026-05-08T09:23:34Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:30]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:30]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2057236919435137024","occurred_at":"2026-05-20T23:07:55Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"42cf30563e6ffe450fb52760384ab78f5961295ed6a6c3e2a7d7dbb0208fe41b","failure_pattern":"UpdateHCPCluster20251223 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-8cb4lx\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-8cb4lx, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-05-14..2026-05-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:223]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000510780\u003e: \n    failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-8cb4lx\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-8cb4lx, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:223]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000510780\u003e: \n    failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-8cb4lx\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-8cb4lx, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2057236919435137024","occurred_at":"2026-05-20T23:07:55Z","signature_id":"74b0f13de19ba98b64f6777ede46843b7a9c90e13827bd1ba32ceca151a76cca","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-05-14..2026-05-20 UTC","prior_windows_present":2,"prior_window_starts":["2026-04-20","2026-05-04"],"prior_runs_affected":8,"prior_last_seen_at":"2026-05-09T09:16:57Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:207]: Unexpected error:\n    \u003c*errors.joinError | 0xc000c80540\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-bqh2d5\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:207]: Unexpected error:\n    \u003c*errors.joinError | 0xc000c80540\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-bqh2d5\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2056970571354214400","occurred_at":"2026-05-20T05:29:32Z","signature_id":"bca1fc92bc5d0e79f3caebd63561b021eb9b229728908e598101c247321039f9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-w8tvcz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-w8tvcz, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-05-14..2026-05-20 UTC","prior_windows_present":4,"prior_window_starts":["2026-04-20","2026-04-27","2026-05-04","2026-05-11"],"prior_runs_affected":29,"prior_last_seen_at":"2026-05-16T00:57:28Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:108]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015895c0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-w8tvcz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-w8tvcz, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:108]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015895c0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-w8tvcz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-w8tvcz, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2057236919435137024","occurred_at":"2026-05-20T23:07:55Z","signature_id":"ad422930f6235b808167096d05244e6b89f30bb8dacdb4055d30a89895105c65","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[0,0,1,0,0,0,1],"trend_range":"2026-05-14..2026-05-20 UTC","prior_windows_present":2,"prior_window_starts":["2026-04-20","2026-05-11"],"prior_runs_affected":5,"prior_last_seen_at":"2026-05-16T08:23:00Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00095e660\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-69q6l.apps.aro.v3s6i0y3r3r2s1o.hzrs.switzerlandnorth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00095e660\u003e: \n    route host was never found: Get \"https://agnhost-e2e-serving-app-69q6l.apps.aro.v3s6i0y3r3r2s1o.hzrs.switzerlandnorth.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2057190193265381376","occurred_at":"2026-05-20T20:02:17Z","signature_id":"c188ff7b4f4d95f830c17d9172077aa263c48d6cc34a628121c4441a8efe654d","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}