{"meta":{"date":"2026-03-28","timezone":"UTC","generated_at":"2026-10-31T23:15:07Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":13,"failed_runs":6,"runs_with_occurrences":6,"runs_with_failure_pattern_matches":6,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":7,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4674/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037984001917980672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4674,"pr_state":"closed","pr_sha":"9e3b1204e642106e1e2759bf6d08ad5c6b897cc2","final_merged_sha":"bf0519515eb8719a71266cf7e1678ca25569f084","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-28T20:03:42Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"664fd36497450dcd3cb28e62b31706adbb4a9c451dad1f2f95b19ec7068cc2fb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4674/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037984001917980672","occurred_at":"2026-03-28T20:03:42Z","failed_at":"e2e","signature_id":"3f002670a7e90ce3e7c3dbe7ff1f1d2aa82c1f96d801f1c9ed77af121931ec11","test_name":"Create HCPOpenShiftCluster with Cilium CNI should create a no-CNI cluster with private etcd and install Cilium","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:173]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc002a68000\u003e: \n    failed to install cilium chart: resource not ready, name: cilium-operator, kind: Deployment, status: InProgress\n    context deadline exceeded\n    {\n        msg: \"failed to install cilium chart: resource not ready, name: cilium-operator, kind: Deployment, status: InProgress\\ncontext deadline exceeded\",\n        err: \u003c*errors.joinError | 0xc0011b4888\u003e{\n            errs: [\n                \u003c*errors.errorString | 0xc0022bc3d0\u003e{\n                    s: \"resource not ready, name: cilium-operator, kind: Deployment, status: InProgress\",\n                },\n                \u003ccontext.deadlineExceededError\u003e{},\n            ],\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:173]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc002a68000\u003e: \n    failed to install cilium chart: resource not ready, name: cilium-operator, kind: Deployment, status: InProgress\n    context deadline exceeded\n    {\n        msg: \"failed to install cilium chart: resource not ready, name: cilium-operator, kind: Deployment, status: InProgress\\ncontext deadline exceeded\",\n        err: \u003c*errors.joinError | 0xc0011b4888\u003e{\n            errs: [\n                \u003c*errors.errorString | 0xc0022bc3d0\u003e{\n                    s: \"resource not ready, name: cilium-operator, kind: Deployment, status: InProgress\",\n                },\n                \u003ccontext.deadlineExceededError\u003e{},\n            ],\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2136fa2e6655fbc68bffdf51efbecc1a8fa59a5f574143c87b971c52c50880cb","failure_pattern":"context deadline exceeded","search_query":"context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2136fa2e6655fbc68bffdf51efbecc1a8fa59a5f574143c87b971c52c50880cb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4673/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037944184257646592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4673,"pr_state":"open","pr_sha":"301d51bdd376019db7ab1bb809ac3201176a174b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-28T17:25:31Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037937270408876032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4672,"pr_state":"closed","pr_sha":"fa52717d3e2211088d479d7b2b713c02a1da86c4","final_merged_sha":"fa52717d3e2211088d479d7b2b713c02a1da86c4","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-28T16:58:02Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4671/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037899703206547456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4671,"pr_state":"closed","pr_sha":"d013cfd86a0c9882a9f89f2672907c05346d93f3","final_merged_sha":"ba582884302572e5b876a2d6e23287a396885bdf","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-28T14:28:46Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037815447864217600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4666,"pr_state":"closed","pr_sha":"1f06de0cd691fb07a53b2eb975128f3b12089291","final_merged_sha":"1f06de0cd691fb07a53b2eb975128f3b12089291","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-28T08:53:55Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4670/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037811582674669568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4670,"pr_state":"closed","pr_sha":"d85b5fed2214e4c4af45c6a078413bb84c72de85","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-28T08:38:36Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4669/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037807363762688000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4669,"pr_state":"closed","pr_sha":"cebfe712ab1fc165ba0352391153c50cb3d32aae","final_merged_sha":"cebfe712ab1fc165ba0352391153c50cb3d32aae","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-28T08:21:50Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4668/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037795162897256448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4668,"pr_state":"closed","pr_sha":"120e03cec727efc0f10e52ec872b1508bbe71789","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-28T07:33:21Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"a8939121d4155b67da0567b47fd431a47e24f4af20268b0a4dd35f9dc2be43e1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4668/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037795162897256448","occurred_at":"2026-03-28T07:33:21Z","failed_at":"provision","signature_id":"4c6336f88cc6168492d88d2f89d6a3eef343dba8237780cdc4c29d6d57aee4a9","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-03-28T08:11:14.140Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e clustersService.image.registry \u003cnil\u003e}:{\u003cnil\u003e clustersService.image.repository \u003cnil\u003e}@{\u003cnil\u003e clustersService.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-28T08:11:19.154Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build05.ci.openshift.org registry.build05.ci.openshift.org registry.build05.ci.openshift.org registry.build05.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image quay.io/raelga/aro-hcp-clusters-service@sha256:4eb6142d00b2a70ad664a70489c343653420fbef712678f2e895e9be7adf17d8 to arohcpsvcdev.azurecr.io/raelga/aro-hcp-clusters-service:4eb6142d00b2a70ad664a70489c343653420fbef712678f2e895e9be7adf17d8.\\nThe image will still be available under it's original digest sha256:4eb6142d00b2a70ad664a70489c343653420fbef712678f2e895e9be7adf17d8 in the target registry.\\nError response from registry: failed to resolve sha256:4eb6142d00b2a70ad664a70489c343653420fbef712678f2e895e9be7adf17d8: unauthorized: access to the requested resource is not authorized: map[]\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"14521338131bd89b6aad8895bf11a69f195096c13fcc40a441b9d1cb3ba5a6e0","failure_pattern":"Error response from registry: failed to resolve sha256:\u003chex\u003e: unauthorized: access to the requested resource is not authorized: map[]","search_query":"Error response from registry: failed to resolve sha256:4eb6142d00b2a70ad664a70489c343653420fbef712678f2e895e9be7adf17d8: unauthorized: access to the requested resource is not authorized: map[]"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["14521338131bd89b6aad8895bf11a69f195096c13fcc40a441b9d1cb3ba5a6e0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037759595757178880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4666,"pr_state":"closed","pr_sha":"809233bb0c951cf404accec90bf11d43057b6b96","final_merged_sha":"1f06de0cd691fb07a53b2eb975128f3b12089291","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-28T05:12:01Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"103c7c941fadead4b6451274c79b3b450042a2aa206a87337380dfed835bba75","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037759595757178880","occurred_at":"2026-03-28T05:12:01Z","failed_at":"e2e","signature_id":"839961bb15b46e8301567278bdb3a6e4bb1e28f3b06aa5d181b787306e244eb1","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001060228\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-5vsdzh\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000c2afa0\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-5vsdzh\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc000c2af60\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-5vsdzh\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000c2af20\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-5vsdzh\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb2b37d0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001060228\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-5vsdzh\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000c2afa0\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-5vsdzh\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc000c2af60\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-5vsdzh\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000c2af20\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-5vsdzh\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb2b37d0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-j9fds8\" to finish deleting: context canceled"}},{"row_id":"2df6977e3415f89d1e39f231c2b77f4feaa645175c64b01f4a389b26ac1f0c9c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037759595757178880","occurred_at":"2026-03-28T05:12:01Z","failed_at":"e2e","signature_id":"24f9a6f2dd0687992c0e269a75066361be9320e749a6267fa2475f342999ad35","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: Timed out after 2400.009s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00101c4e0\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.16) failed: no version in clusterversion/version status.history is greater than initial \"4.20.16\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000b00c20\u003e{\n                msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.16) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.16\\\"\",\n                err: \u003c*errors.errorString | 0xc000804560\u003e{\n                    s: \"no version in clusterversion/version status.history is greater than initial \\\"4.20.16\\\"\",\n                },\n            },\n        ],\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: Timed out after 2400.009s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00101c4e0\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.16) failed: no version in clusterversion/version status.history is greater than initial \"4.20.16\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000b00c20\u003e{\n                msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.16) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.16\\\"\",\n                err: \u003c*errors.errorString | 0xc000804560\u003e{\n                    s: \"no version in clusterversion/version status.history is greater than initial \\\"4.20.16\\\"\",\n                },\n            },\n        ],\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4"}},{"row_id":"5046c773f358cdf0a7aaa5043bfc30076aa72f02be9d78f214044702f31c3b7a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037759595757178880","occurred_at":"2026-03-28T05:12:01Z","failed_at":"e2e","signature_id":"c6e335d05524d381f10449bca2c2a2fafea858c66e367b0bb70ba5b6a50fe53e","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: Timed out after 2400.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00100f2f0\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.6) failed: no version in clusterversion/version status.history is greater than initial \"4.21.6\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000956ee0\u003e{\n                msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.6) failed: no version in clusterversion/version status.history is greater than initial \\\"4.21.6\\\"\",\n                err: \u003c*errors.errorString | 0xc0012f65e0\u003e{\n                    s: \"no version in clusterversion/version status.history is greater than initial \\\"4.21.6\\\"\",\n                },\n            },\n        ],\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: Timed out after 2400.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00100f2f0\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.6) failed: no version in clusterversion/version status.history is greater than initial \"4.21.6\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000956ee0\u003e{\n                msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.6) failed: no version in clusterversion/version status.history is greater than initial \\\"4.21.6\\\"\",\n                err: \u003c*errors.errorString | 0xc0012f65e0\u003e{\n                    s: \"no version in clusterversion/version status.history is greater than initial \\\"4.21.6\\\"\",\n                },\n            },\n        ],\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4"}},{"row_id":"8762450fe3d80ec30873dee0ecb76850a2f2c42cdfa1642bd9859bb5007c4b8a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037759595757178880","occurred_at":"2026-03-28T05:12:01Z","failed_at":"e2e","signature_id":"f06316a757984ef34d2296a369741db7580c0c284a99c68393b1c045bac3cd4d","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013cc180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-j9fds8\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000b0df20\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-j9fds8\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc000b0df00\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-j9fds8\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000b0dee0\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-j9fds8\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb2b37d0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013cc180\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-j9fds8\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000b0df20\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-j9fds8\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc000b0df00\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-j9fds8\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000b0dee0\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-j9fds8\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb2b37d0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-j9fds8\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4639/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037759478257946624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4639,"pr_state":"closed","pr_sha":"ea1e30499989c3e372f9a6bd359924e00d95b757","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-28T05:11:33Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037730518652424192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4665,"pr_state":"closed","pr_sha":"965139d68e85cbd36fb3fb0c1645c8da6e61dbf8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-28T03:16:29Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"2cdc7ca0c3b5cc0b5e148e2aa5df42a63bea707e9c04953acb5a9b6867702896","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037730518652424192","occurred_at":"2026-03-28T03:16:29Z","failed_at":"e2e","signature_id":"8205d99cb3e1ec451d8f3ee0dd11d0fdc6c352879933497721cbfd204fcd8133","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00017e150\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-8s5fww\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000969680\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-8s5fww\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc000969660\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-8s5fww\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000969640\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-8s5fww\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb2b17d0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00017e150\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-8s5fww\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000969680\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-8s5fww\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc000969660\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-8s5fww\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000969640\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-8s5fww\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb2b17d0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-j9fds8\" to finish deleting: context canceled"}},{"row_id":"bddd21a3407b6debabda16bc3ce56ddb50f2880bbad6dd6c2a74aeb8c6ce1fa7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037730518652424192","occurred_at":"2026-03-28T03:16:29Z","failed_at":"e2e","signature_id":"2c3f2122d1acfdaf0a393521cbf9d6bc73774ee07e8ca420cc9ca84f1f527cc7","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: Timed out after 2400.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0013a6570\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.16) failed: no version in clusterversion/version status.history is greater than initial \"4.20.16\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc001449840\u003e{\n                msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.16) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.16\\\"\",\n                err: \u003c*errors.errorString | 0xc0007bce30\u003e{\n                    s: \"no version in clusterversion/version status.history is greater than initial \\\"4.20.16\\\"\",\n                },\n            },\n        ],\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: Timed out after 2400.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0013a6570\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.16) failed: no version in clusterversion/version status.history is greater than initial \"4.20.16\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc001449840\u003e{\n                msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.16) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.16\\\"\",\n                err: \u003c*errors.errorString | 0xc0007bce30\u003e{\n                    s: \"no version in clusterversion/version status.history is greater than initial \\\"4.20.16\\\"\",\n                },\n            },\n        ],\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4"}},{"row_id":"e86ddd6b874e573426289e9e04d083fcc936cda5f07d926df24debbc2cdbdef9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037730518652424192","occurred_at":"2026-03-28T03:16:29Z","failed_at":"e2e","signature_id":"7b7b34649c57df46deb3f48a0a63c687908039a423acc01dc4cad23cb08feb99","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: Timed out after 2400.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000d42438\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.6) failed: no version in clusterversion/version status.history is greater than initial \"4.21.6\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000a81540\u003e{\n                msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.6) failed: no version in clusterversion/version status.history is greater than initial \\\"4.21.6\\\"\",\n                err: \u003c*errors.errorString | 0xc00073b050\u003e{\n                    s: \"no version in clusterversion/version status.history is greater than initial \\\"4.21.6\\\"\",\n                },\n            },\n        ],\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: Timed out after 2400.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000d42438\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.6) failed: no version in clusterversion/version status.history is greater than initial \"4.21.6\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000a81540\u003e{\n                msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.6) failed: no version in clusterversion/version status.history is greater than initial \\\"4.21.6\\\"\",\n                err: \u003c*errors.errorString | 0xc00073b050\u003e{\n                    s: \"no version in clusterversion/version status.history is greater than initial \\\"4.21.6\\\"\",\n                },\n            },\n        ],\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037704519084675072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4666,"pr_state":"closed","pr_sha":"809233bb0c951cf404accec90bf11d43057b6b96","final_merged_sha":"1f06de0cd691fb07a53b2eb975128f3b12089291","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-28T01:33:10Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"032261812cc90795e9ab2f7134b46d829853b0f7d66ca5aff227a9ee988fbc29","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037704519084675072","occurred_at":"2026-03-28T01:33:10Z","failed_at":"e2e","signature_id":"68a48af49cf15d1cadb0778b89e53280097463070ed6a99d61ab5183f80b9275","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001120360\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-w2p7hf\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000d8a280\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-w2p7hf\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc000d8a260\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-w2p7hf\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000d8a240\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-w2p7hf\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb2b37d0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001120360\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-w2p7hf\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000d8a280\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-w2p7hf\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc000d8a260\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-w2p7hf\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000d8a240\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-w2p7hf\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb2b37d0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-j9fds8\" to finish deleting: context canceled"}},{"row_id":"b05a78afb810fd683a2c0551e121ebb7f1ca1b494e1ead52a675c78c3da406a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037704519084675072","occurred_at":"2026-03-28T01:33:10Z","failed_at":"e2e","signature_id":"04900cab327676740613393500e23c1551d1b31966df591126176e40c28200d2","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: Timed out after 2400.004s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0011fb800\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.16) failed: no version in clusterversion/version status.history is greater than initial \"4.20.16\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000828580\u003e{\n                msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.16) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.16\\\"\",\n                err: \u003c*errors.errorString | 0xc000a2c320\u003e{\n                    s: \"no version in clusterversion/version status.history is greater than initial \\\"4.20.16\\\"\",\n                },\n            },\n        ],\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: Timed out after 2400.004s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0011fb800\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.16) failed: no version in clusterversion/version status.history is greater than initial \"4.20.16\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000828580\u003e{\n                msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.16) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.16\\\"\",\n                err: \u003c*errors.errorString | 0xc000a2c320\u003e{\n                    s: \"no version in clusterversion/version status.history is greater than initial \\\"4.20.16\\\"\",\n                },\n            },\n        ],\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4"}},{"row_id":"b682290b5a3fff193683f5424e2293027330965e1ca3d5324e912ea33e29a798","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4666/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037704519084675072","occurred_at":"2026-03-28T01:33:10Z","failed_at":"e2e","signature_id":"7a7b77c4664659c8fdecbda8a26aa6cec1803bb16b711ac05f2d381892584f47","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: Timed out after 2400.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000ffd110\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.6) failed: no version in clusterversion/version status.history is greater than initial \"4.21.6\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000d81c80\u003e{\n                msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.6) failed: no version in clusterversion/version status.history is greater than initial \\\"4.21.6\\\"\",\n                err: \u003c*errors.errorString | 0xc0007aae10\u003e{\n                    s: \"no version in clusterversion/version status.history is greater than initial \\\"4.21.6\\\"\",\n                },\n            },\n        ],\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: Timed out after 2400.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000ffd110\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.6) failed: no version in clusterversion/version status.history is greater than initial \"4.21.6\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000d81c80\u003e{\n                msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.6) failed: no version in clusterversion/version status.history is greater than initial \\\"4.21.6\\\"\",\n                err: \u003c*errors.errorString | 0xc0007aae10\u003e{\n                    s: \"no version in clusterversion/version status.history is greater than initial \\\"4.21.6\\\"\",\n                },\n            },\n        ],\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037702714476990464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4665,"pr_state":"closed","pr_sha":"965139d68e85cbd36fb3fb0c1645c8da6e61dbf8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-28T01:26:00Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"033a12edac692701682df75c20bfbbbf3fc78bbe699c13c0dcc5da865eb954a8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037702714476990464","occurred_at":"2026-03-28T01:26:00Z","failed_at":"e2e","signature_id":"27dcf02bc1275b9c227dea5f3ba21b93a6506a610992f187a24c545dfa5e6cc1","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001264240\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-lnw9t5\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0009b9520\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-lnw9t5\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc0009b94e0\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-lnw9t5\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc0009b94c0\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-lnw9t5\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb2b17d0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001264240\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-lnw9t5\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0009b9520\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-lnw9t5\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc0009b94e0\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-lnw9t5\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc0009b94c0\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-lnw9t5\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb2b17d0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-j9fds8\" to finish deleting: context canceled"}},{"row_id":"6fa79d0a5d6b349a3657e2b10a093a99ca0819abe2768facae045f7b018f0d4e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037702714476990464","occurred_at":"2026-03-28T01:26:00Z","failed_at":"e2e","signature_id":"1c701cd03894d77baabf7be55f2b628a090e310d6009a0b2db237f8ca3549a92","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: Timed out after 2400.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00108a5d0\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.16) failed: no version in clusterversion/version status.history is greater than initial \"4.20.16\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000e04620\u003e{\n                msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.16) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.16\\\"\",\n                err: \u003c*errors.errorString | 0xc000a38830\u003e{\n                    s: \"no version in clusterversion/version status.history is greater than initial \\\"4.20.16\\\"\",\n                },\n            },\n        ],\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: Timed out after 2400.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00108a5d0\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.16) failed: no version in clusterversion/version status.history is greater than initial \"4.20.16\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000e04620\u003e{\n                msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.16) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.16\\\"\",\n                err: \u003c*errors.errorString | 0xc000a38830\u003e{\n                    s: \"no version in clusterversion/version status.history is greater than initial \\\"4.20.16\\\"\",\n                },\n            },\n        ],\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4"}},{"row_id":"d5053ab69acd1c969c95326c0ba0acb4bd3346072b271f0b5837ff664501d736","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037702714476990464","occurred_at":"2026-03-28T01:26:00Z","failed_at":"e2e","signature_id":"e7c94f3b0f91aeaf536302406df5441f891f0f6f802f47f6f9f16896588bb9fd","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: Timed out after 2400.009s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0012356b0\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.6) failed: no version in clusterversion/version status.history is greater than initial \"4.21.6\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0009ee6c0\u003e{\n                msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.6) failed: no version in clusterversion/version status.history is greater than initial \\\"4.21.6\\\"\",\n                err: \u003c*errors.errorString | 0xc0008e03b0\u003e{\n                    s: \"no version in clusterversion/version status.history is greater than initial \\\"4.21.6\\\"\",\n                },\n            },\n        ],\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:122]: Timed out after 2400.009s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0012356b0\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.6) failed: no version in clusterversion/version status.history is greater than initial \"4.21.6\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0009ee6c0\u003e{\n                msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.6) failed: no version in clusterversion/version status.history is greater than initial \\\"4.21.6\\\"\",\n                err: \u003c*errors.errorString | 0xc0008e03b0\u003e{\n                    s: \"no version in clusterversion/version status.history is greater than initial \\\"4.21.6\\\"\",\n                },\n            },\n        ],\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}