{"meta":{"date":"2026-06-09","timezone":"UTC","generated_at":"2026-08-29T07:17:05Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":12,"failed_runs":10,"runs_with_occurrences":10,"runs_with_failure_pattern_matches":10,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":2,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2064482691524333568","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-09T23:00:04Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"352fa043fd6cddf2ab943b2d3f53c78816d8271b60deb79ce33ca6845a19440c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2064482691524333568","occurred_at":"2026-06-09T23:00:04Z","failed_at":"e2e","signature_id":"3a3c6f7f498c1a582b3535b57f14b8795faefffe30fdd73a30f7217871698430","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:704]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001040b40\u003e: \n    failed to create HCP cluster np-dg-minor-95jzkb: failed starting cluster creation \"np-dg-minor-95jzkb\" in resourcegroup=\"rg-np-dg-minor-95jzkb-g9rpcw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-95jzkb-g9rpcw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-95jzkb\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.21.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:704]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001040b40\u003e: \n    failed to create HCP cluster np-dg-minor-95jzkb: failed starting cluster creation \"np-dg-minor-95jzkb\" in resourcegroup=\"rg-np-dg-minor-95jzkb-g9rpcw\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-95jzkb-g9rpcw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-95jzkb\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.21.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"90e9ab5a508c69fa469c74ea9a9d435aee1bda7d3791c7e7ad6b5d426a15064f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"cb00bbbc8e894b48624311a6fb1096a18ce7726b663d8b7cb5a8482baa28cb61","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2064482691524333568","occurred_at":"2026-06-09T23:00:04Z","failed_at":"e2e","signature_id":"6fa9bd64410cbb9e76117cebe3f64e3a4d5983b74baa2fcf9a0d73959c84ad01","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-f684mv at version 4.22.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d63120\u003e: \n    failed to create HCP cluster np-skip-minor-f684mv: failed starting cluster creation \"np-skip-minor-f684mv\" in resourcegroup=\"rg-np-skip-minor-f684mv-hgkqrz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-f684mv-hgkqrz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-f684mv\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.22.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-f684mv at version 4.22.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d63120\u003e: \n    failed to create HCP cluster np-skip-minor-f684mv: failed starting cluster creation \"np-skip-minor-f684mv\" in resourcegroup=\"rg-np-skip-minor-f684mv-hgkqrz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-f684mv-hgkqrz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-f684mv\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.22.0-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"90e9ab5a508c69fa469c74ea9a9d435aee1bda7d3791c7e7ad6b5d426a15064f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v\u003cversion\u003e' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["90e9ab5a508c69fa469c74ea9a9d435aee1bda7d3791c7e7ad6b5d426a15064f"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064444081047605248","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-09T19:39:47Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"ba1c72f292b0dbab89a8a8b460e0a4c39db6008e0069fbf6ef5694ae95ae5aa3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064444081047605248","occurred_at":"2026-06-09T19:39:47Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064436508076544000","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-09T19:29:02Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"9e52a87dff4b9cd9c3774e3931de8c5d1978f9a05e2ed8d69df40487201e83e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064436508076544000","occurred_at":"2026-06-09T19:29:02Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"a2376cc2e8ab9d5ab35a7205a797bdc2cba470cd9bc56ecc6ff5a67d366c050c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064436508076544000","occurred_at":"2026-06-09T19:29:02Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064425150199107584","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-09T19:11:25Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f1b0586c89694aec6e3819c5e25829c46d2e5445682876c6dfd5c99de3124025","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064425150199107584","occurred_at":"2026-06-09T19:11:25Z","failed_at":"e2e","signature_id":"7e509204663de1041322d583434342eaadafda17d51e7eeda1a39e99962f3e92","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-vmsnfk/cluster-nightly-5-0-6q4lgj should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008e6870\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-6q4lgj\" in resourcegroup=\"rg-nightly-5-0-vmsnfk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-6q4lgj in resource group rg-nightly-5-0-vmsnfk, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-vmsnfk/cluster-nightly-5-0-6q4lgj should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008e6870\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-6q4lgj\" in resourcegroup=\"rg-nightly-5-0-vmsnfk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-6q4lgj in resource group rg-nightly-5-0-vmsnfk, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064424644064055296","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-06-09T19:09:24Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064396799321313280","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-09T17:18:46Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"3252b5f2f3d7039bcc88760b63f4e566a0bd96a6ad4058ab714ee5a0699b71b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064396799321313280","occurred_at":"2026-06-09T17:18:46Z","failed_at":"e2e","signature_id":"a55b64c48f3248d8ce4b52cb22e6d907463e0ca467c2a061f63cd84d751b39a7","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-c2nxww/cluster-nightly-5-0-vhm589 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00150c4e0\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-vhm589\" in resourcegroup=\"rg-nightly-5-0-c2nxww\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-vhm589 in resource group rg-nightly-5-0-c2nxww, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-c2nxww/cluster-nightly-5-0-vhm589 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00150c4e0\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-vhm589\" in resourcegroup=\"rg-nightly-5-0-c2nxww\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-vhm589 in resource group rg-nightly-5-0-c2nxww, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded"}},{"row_id":"67603095bc4d992a79d04fcea85435446e3f3aa5e609904f9d80ef355fe61351","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064396799321313280","occurred_at":"2026-06-09T17:18:46Z","failed_at":"e2e","signature_id":"595ea2a0b0a16cae02d995131ef64eb25872da4fa709c4c4d7214dc5710cdb52","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-vcfqbx/cluster-nightly-4-23-fnbphq should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001268690\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-fnbphq\" in resourcegroup=\"rg-nightly-4-23-vcfqbx\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-fnbphq in resource group rg-nightly-4-23-vcfqbx, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-vcfqbx/cluster-nightly-4-23-fnbphq should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001268690\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-fnbphq\" in resourcegroup=\"rg-nightly-4-23-vcfqbx\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-fnbphq in resource group rg-nightly-4-23-vcfqbx, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064396319065116672","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-09T17:16:51Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"238f24255e21acf6583321288b81dc8cc9e4d5ab7870b64928166dcd83ba7408","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064396319065116672","occurred_at":"2026-06-09T17:16:51Z","failed_at":"e2e","signature_id":"c40934efb8a0c36d54d9ac7fc3aa5478fd79b13a78f23812cb1fc060ac6db64c","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009485a0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-kkjnrm\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group complex-cilium-kv-kkjnrm, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009485a0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-kkjnrm\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group complex-cilium-kv-kkjnrm, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded"}},{"row_id":"2b3f356ac963b27ae253f3faa73db4d5cd9b40ff988c96487bca3f3a738d3e28","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064396319065116672","occurred_at":"2026-06-09T17:16:51Z","failed_at":"e2e","signature_id":"349f1814f2668e5b71ebf02b93d024e87eb5a3d72775bf25646ae00f9397c20e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-9s2nzg/cluster-nightly-4-23-5jn2lq should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001370930\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-5jn2lq\" in resourcegroup=\"rg-nightly-4-23-9s2nzg\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-5jn2lq in resource group rg-nightly-4-23-9s2nzg, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-9s2nzg/cluster-nightly-4-23-5jn2lq should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001370930\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-5jn2lq\" in resourcegroup=\"rg-nightly-4-23-9s2nzg\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-5jn2lq in resource group rg-nightly-4-23-9s2nzg, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded"}},{"row_id":"3db97a5da473dfbd72c87cdd410ce1b98cbfce60672b605706fee8f2314e5498","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064396319065116672","occurred_at":"2026-06-09T17:16:51Z","failed_at":"e2e","signature_id":"38de80d8706b44c08c01b2a8c363d15110ae600984ec9536a1b9d24218e36fdb","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-szhbft/cluster-nightly-5-0-t5fcjl should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001165c20\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-t5fcjl\" in resourcegroup=\"rg-nightly-5-0-szhbft\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-t5fcjl in resource group rg-nightly-5-0-szhbft, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-szhbft/cluster-nightly-5-0-t5fcjl should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001165c20\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-t5fcjl\" in resourcegroup=\"rg-nightly-5-0-szhbft\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-t5fcjl in resource group rg-nightly-5-0-szhbft, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded"}},{"row_id":"79b31fab72b404e498da1be8b71827284a2922359692bab19d5b49509c2fb402","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064396319065116672","occurred_at":"2026-06-09T17:16:51Z","failed_at":"e2e","signature_id":"e64ab0717150294a49d4e303bf7f7c2d51c7972d8b7f0fade88f4fa311044600","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:136]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009906f0\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-852q76, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-852q76\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:136]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009906f0\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-852q76, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-852q76\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-852q76, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-852q76\" to finish creating: context deadline exceeded"}},{"row_id":"df612aa31e53443acea3357c735ffd8baa543e4749572f1fc2cdb51ec9e1963d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064396319065116672","occurred_at":"2026-06-09T17:16:51Z","failed_at":"e2e","signature_id":"526b0473053428aa7590fc58282c30611713aa9a50937f90d7e0ae5d8dacb7ee","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-ggxw2n\" for cluster \"cluster-candidate-4-21-ggxw2n\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0005d87b0\u003e: \n    failed to create NodePool np-ggxw2n, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-ggxw2n in resource group rg-candidate-4-21-chmwt8, error: failed waiting for nodepool=\"np-ggxw2n\" for cluster \"cluster-candidate-4-21-ggxw2n\" in resourcegroup=\"rg-candidate-4-21-chmwt8\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-ggxw2n\" for cluster \"cluster-candidate-4-21-ggxw2n\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0005d87b0\u003e: \n    failed to create NodePool np-ggxw2n, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-ggxw2n in resource group rg-candidate-4-21-chmwt8, error: failed waiting for nodepool=\"np-ggxw2n\" for cluster \"cluster-candidate-4-21-ggxw2n\" in resourcegroup=\"rg-candidate-4-21-chmwt8\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-852q76, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-852q76\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064316817584689152","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-06-09T12:00:18Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064310885882531840","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-09T11:37:22Z"},"failed_at":["e2e"],"failed_test_count":9,"occurrences":[{"row_id":"0ad10c4c964ef3cc8d0261f92129419033c8c2abb121f01cf1640d77cc576778","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064310885882531840","occurred_at":"2026-06-09T11:37:22Z","failed_at":"e2e","signature_id":"844a0be22d6f0db0efb47416f9afc1192a2df91b608c90b252fa8fcc2525c42b","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:89]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00109eb40\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-675sc9, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-675sc9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-675sc9, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:89]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00109eb40\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-675sc9, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-675sc9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-675sc9, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded"}},{"row_id":"66cc9b4bacfc7cfe458efefb4a710aeeaf4831be15c95a233cad2024e66d12a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064310885882531840","occurred_at":"2026-06-09T11:37:22Z","failed_at":"e2e","signature_id":"3705190eecfbba1c9ea51d9eec1551324cdc8eb274b9bde829ead5b6966e2e3a","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001602600\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-2cm842\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-2cm842, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001602600\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-2cm842\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-2cm842, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded"}},{"row_id":"87c8eee9df4852bff51762c4bdfdfe134e28b22d5b63ebe057273d6ed33eab4b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064310885882531840","occurred_at":"2026-06-09T11:37:22Z","failed_at":"e2e","signature_id":"7653ffd4e02735b234dd652c0dd32332c9566363ef7806113eb84037d3ada7a9","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 1200.004s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 1200.004s.\nCluster should become ready within '20.000000' minutes\nExpected\n    \u003cbool\u003e: ...\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dcad60eb8db626db10bfb0812a18e7c392049987a81c4fcbcdc5140e0032dc41","failure_pattern":"Cluster should become ready within '20.000000' minutes","search_query":"Cluster should become ready within '20.000000' minutes"}},{"row_id":"9bb8ca3863a0810d79c1387f4cefbd70eb30c66b8cf746d0bb347996dae0a997","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064310885882531840","occurred_at":"2026-06-09T11:37:22Z","failed_at":"e2e","signature_id":"5515548644808dca6ae2c15348ba82a72b566ec2132d29fabad81f437d546dfe","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-mx2rw2/cluster-nightly-4-23-clnb9r should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001074ab0\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-clnb9r\" in resourcegroup=\"rg-nightly-4-23-mx2rw2\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-clnb9r in resource group rg-nightly-4-23-mx2rw2, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-mx2rw2/cluster-nightly-4-23-clnb9r should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001074ab0\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-clnb9r\" in resourcegroup=\"rg-nightly-4-23-mx2rw2\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-clnb9r in resource group rg-nightly-4-23-mx2rw2, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded"}},{"row_id":"a2d8d619a322407ae9822e49e20e76064f974aa272a0e764b741fd7f7a1cb8bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064310885882531840","occurred_at":"2026-06-09T11:37:22Z","failed_at":"e2e","signature_id":"928a47cb7944bb323bec31fa2a5eb2975c526028b36598f2c149cce344dc83a6","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:83]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f01710\u003e: \n    failed to create HCP cluster patch-name-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-pp74jp, error: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-pp74jp\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-pp74jp, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:83]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f01710\u003e: \n    failed to create HCP cluster patch-name-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-pp74jp, error: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-pp74jp\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-pp74jp, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded"}},{"row_id":"cc01ebfe5b9383ed9d533d80260ba0ccffc74fff189f6151344f7098c3db3827","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064310885882531840","occurred_at":"2026-06-09T11:37:22Z","failed_at":"e2e","signature_id":"3b2c8703c3d23bcc2f6a7689b763841419941d28dedafe3f9b4b08f22e36bc47","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:199]: Timed out after 2520.005s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:197 with:\ncluster has not yet reached Succeeded state\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nto equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:199]: Timed out after 2520.005s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:197 with:\ncluster has not yet reached Succeeded state\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nto equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"471f22e5967d4c6d260358df80d5f1c22cdca60ff8d89437ff00486c66a994f2","failure_pattern":"cluster has not yet reached Succeeded state","search_query":"cluster has not yet reached Succeeded state"}},{"row_id":"d18fcbe50b757113548c82047cd7ff1af180ae93b35632240954d2b600eacef4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064310885882531840","occurred_at":"2026-06-09T11:37:22Z","failed_at":"e2e","signature_id":"a96e9bb6a3c1c06e294ca56e88c2ee62c54caa2ef50ccd1e23b8595b7e5e7d09","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-vf9czb/cluster-nightly-5-0-jk5fg7 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000fc96b0\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-jk5fg7\" in resourcegroup=\"rg-nightly-5-0-vf9czb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-jk5fg7 in resource group rg-nightly-5-0-vf9czb, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-vf9czb/cluster-nightly-5-0-jk5fg7 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000fc96b0\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-jk5fg7\" in resourcegroup=\"rg-nightly-5-0-vf9czb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-jk5fg7 in resource group rg-nightly-5-0-vf9czb, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded"}},{"row_id":"e5f1cf6d9f338cc75213a6ad2f27ba0fc09c7c59ddf6d8b2bc3e2930cd7278b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064310885882531840","occurred_at":"2026-06-09T11:37:22Z","failed_at":"e2e","signature_id":"fb044cc30a88a2114dc710251c14a97955fa8c99dd37770d29a352a18e9abce8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:205]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-qrwxz9\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000997620\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-92ldj.apps.aro.r8i7j7z8f3c5v9h.00gq.eastus2euap.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:205]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-qrwxz9\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000997620\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-92ldj.apps.aro.r8i7j7z8f3c5v9h.00gq.eastus2euap.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"e991cdf1365fb6b10510119d991df1a2ef4b1a3cd1bf08f1312f3f6b8edb6dc7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064310885882531840","occurred_at":"2026-06-09T11:37:22Z","failed_at":"e2e","signature_id":"ade999842ddd3e74c5c690b7471eb7ba8f12f7bd7de77ead056f002b96d49d4f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-hzgwbr\" for cluster \"cluster-candidate-4-21-hzgwbr\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001293e30\u003e: \n    failed to create NodePool np-hzgwbr, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-hzgwbr in resource group rg-candidate-4-21-9d9sd9, error: failed waiting for nodepool=\"np-hzgwbr\" for cluster \"cluster-candidate-4-21-hzgwbr\" in resourcegroup=\"rg-candidate-4-21-9d9sd9\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-hzgwbr\" for cluster \"cluster-candidate-4-21-hzgwbr\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001293e30\u003e: \n    failed to create NodePool np-hzgwbr, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-hzgwbr in resource group rg-candidate-4-21-9d9sd9, error: failed waiting for nodepool=\"np-hzgwbr\" for cluster \"cluster-candidate-4-21-hzgwbr\" in resourcegroup=\"rg-candidate-4-21-9d9sd9\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-852q76, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-852q76\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":9,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","471f22e5967d4c6d260358df80d5f1c22cdca60ff8d89437ff00486c66a994f2","c6cc39ebab92c7ed45d8097741d416d66b29b3bacdc947677b26088c375b717d","dcad60eb8db626db10bfb0812a18e7c392049987a81c4fcbcdc5140e0032dc41"],"matched_occurrences":9,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064279856266350592","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-09T09:34:04Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"da6c1f287c4f32d61e4659099bb4bb342e6c54eb5d51fb9de2850d0d293d1616","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064279856266350592","occurred_at":"2026-06-09T09:34:04Z","failed_at":"e2e","signature_id":"88910cff0ca3e97103f1cf61ad68d8d3d30060e5c912607a89e67adbc2ff2314","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00132ee10\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-2s6t8f, error: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-2s6t8f\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-2s6t8f, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00132ee10\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-2s6t8f, error: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-2s6t8f\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-2s6t8f, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064264482309279744","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-09T08:32:59Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"ca145aaccb6f968425846f7af788772ac0974016b21bcf0c4cf567573a77232f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2064264482309279744","occurred_at":"2026-06-09T08:32:59Z","failed_at":"e2e","signature_id":"9a3c9f282540cd44f49cf8c739dc0fa1b8fe77a908fb12e831a3b72f463c3da7","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0004fc690\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-r7ldlr, error: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-r7ldlr\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-r7ldlr, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0004fc690\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-r7ldlr, error: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-r7ldlr\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-r7ldlr, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-06-09T02:00:03Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"1d7ed32a56c3ec2ce884e1907013ed2074ea468ba047557ae573598ce53d74d7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","occurred_at":"2026-06-09T02:00:03Z","failed_at":"e2e","signature_id":"cff2360eac2cad168884885eaabbbf173a7621bebc74e33d785976a60f7c9db8","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014de450\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-5whn7c\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-5whn7c, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: failed to create HCP cluster version 4.19\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014de450\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-5whn7c\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-5whn7c, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded"}},{"row_id":"25c7dc609f3b410b17188c454d56131b0e8d4e967c45bc9a17f1bbaf6828564e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","occurred_at":"2026-06-09T02:00:03Z","failed_at":"e2e","signature_id":"c7a5e8940abe777af1144339954157b679d83e1470af1aaa49f44ae6d47592b2","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137]: failed to create HCP cluster \"cluster-zstream-4-20-cqlcfn\" with version 4.20.23 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001900c90\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-cqlcfn, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-cqlcfn in resource group rg-zstream-upgrade-4-20-ft2rr4, error: failed waiting for cluster=\"cluster-zstream-4-20-cqlcfn\" in resourcegroup=\"rg-zstream-upgrade-4-20-ft2rr4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-cqlcfn in resource group rg-zstream-upgrade-4-20-ft2rr4, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137]: failed to create HCP cluster \"cluster-zstream-4-20-cqlcfn\" with version 4.20.23 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001900c90\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-cqlcfn, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-cqlcfn in resource group rg-zstream-upgrade-4-20-ft2rr4, error: failed waiting for cluster=\"cluster-zstream-4-20-cqlcfn\" in resourcegroup=\"rg-zstream-upgrade-4-20-ft2rr4\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-cqlcfn in resource group rg-zstream-upgrade-4-20-ft2rr4, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded"}},{"row_id":"5cd8f5bad53e261c2df3d069d1cf161b5566d25951c1bc9a71cebf2a726e5c6c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","occurred_at":"2026-06-09T02:00:03Z","failed_at":"e2e","signature_id":"abd0fc63fd2db9f860e7eb6fe246b2d00a034d5b3c3244a18aad6858b101bf25","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:112]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00159a6c0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-rs9vtl, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-rs9vtl\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-rs9vtl, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:112]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00159a6c0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-rs9vtl, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-rs9vtl\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-rs9vtl, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded"}},{"row_id":"5f735989ade1b8df10ff098e0d69b628bdfcb2ec7492f5bcbdc41681863c5adc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","occurred_at":"2026-06-09T02:00:03Z","failed_at":"e2e","signature_id":"4b2fdb0b9028b543839c0319632f4da5cc1da42c264cf1ea145a9c68e3211b72","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:137]: HCP cluster rg-candidate-4-21-7g8djb/cluster-candidate-4-21-xmtssq should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0000eae10\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:137]: HCP cluster rg-candidate-4-21-7g8djb/cluster-candidate-4-21-xmtssq should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0000eae10\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded"}},{"row_id":"6389ca6bf0f8df842794dcbfd9d262999a6604d569bb74c1a13be0fbec1d550f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","occurred_at":"2026-06-09T02:00:03Z","failed_at":"e2e","signature_id":"a0304bca149abe4d70e3e12d31cba0a08934172e0298275e0c3c321a78013adf","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-b5kj4q/cluster-nightly-4-23-c6nrfd should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010aa330\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-c6nrfd\" in resourcegroup=\"rg-nightly-4-23-b5kj4q\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-c6nrfd in resource group rg-nightly-4-23-b5kj4q, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-b5kj4q/cluster-nightly-4-23-c6nrfd should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010aa330\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-c6nrfd\" in resourcegroup=\"rg-nightly-4-23-b5kj4q\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-c6nrfd in resource group rg-nightly-4-23-b5kj4q, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded"}},{"row_id":"8116da6d859eeeaf41d3625a39375be0c43e2a91b0766e6d0c55d70cb4c68ed9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","occurred_at":"2026-06-09T02:00:03Z","failed_at":"e2e","signature_id":"d090d8b072b52dae181e782adaae6dce5b5171c994ae4865ad600637c423eeea","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:137]: HCP cluster rg-candidate-4-22-bhks5n/cluster-candidate-4-22-np4fcf should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00180f320\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-np4fcf, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-np4fcf in resource group rg-candidate-4-22-bhks5n, error: failed waiting for cluster=\"cluster-candidate-4-22-np4fcf\" in resourcegroup=\"rg-candidate-4-22-bhks5n\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-np4fcf in resource group rg-candidate-4-22-bhks5n, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:137]: HCP cluster rg-candidate-4-22-bhks5n/cluster-candidate-4-22-np4fcf should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00180f320\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-np4fcf, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-np4fcf in resource group rg-candidate-4-22-bhks5n, error: failed waiting for cluster=\"cluster-candidate-4-22-np4fcf\" in resourcegroup=\"rg-candidate-4-22-bhks5n\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-np4fcf in resource group rg-candidate-4-22-bhks5n, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded"}},{"row_id":"a2e3b0f853b88b363a99cb11ac3d0c2e9eb0841af6a6cb06b86496a23c215d8e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","occurred_at":"2026-06-09T02:00:03Z","failed_at":"e2e","signature_id":"b51928eb0bc2acc1b2df6c1f7efd4a6f5eba2e47b73c6478484af03364201334","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0002b8420\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-bzsg8d\" resourcegroup=\"rg-np-dg-minor-bzsg8d-gwv8l6\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-bzsg8d-gwv8l6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-bzsg8d/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": invalid node pool version 4.19.33: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": cannot downgrade from version 4.21.18 to 4.19.33\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:750]: failed to update nodepool npdg-421-419 to downgrade target 4.19.33\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0002b8420\u003e: \n    failed to start nodepool \"npdg-421-419\" update in cluster \"np-dg-minor-bzsg8d\" resourcegroup=\"rg-np-dg-minor-bzsg8d-gwv8l6\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-bzsg8d-gwv8l6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-bzsg8d/nodePools/npdg-421-419\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": invalid node pool version 4.19.33: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.19.33\\\": cannot downgrade from version 4.21.18 to 4.19.33\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"53fa6318769bb2a786556b3fac950c44b85ab8ba4f6fa60b2949a759bcc916c1","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.19.33\": cannot downgrade from version 4.21.18 to 4.19.33; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"cbedd1621fbf1e2fddbbbb8751dee7dec8a521e85d7c9c6e0684d7cb7d441c66","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","occurred_at":"2026-06-09T02:00:03Z","failed_at":"e2e","signature_id":"7a19e46774a7fcfb8f26f0a8ca344c40ed042d97c7b5b00d009abeb7e1f1fe79","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137]: failed to create HCP cluster \"cluster-zstream-4-22-phvfxj\" with version 4.22.0-rc.4 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d1af00\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-phvfxj, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-phvfxj in resource group rg-zstream-upgrade-4-22-g9lt82, error: failed waiting for cluster=\"cluster-zstream-4-22-phvfxj\" in resourcegroup=\"rg-zstream-upgrade-4-22-g9lt82\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-phvfxj in resource group rg-zstream-upgrade-4-22-g9lt82, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137]: failed to create HCP cluster \"cluster-zstream-4-22-phvfxj\" with version 4.22.0-rc.4 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d1af00\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-phvfxj, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-phvfxj in resource group rg-zstream-upgrade-4-22-g9lt82, error: failed waiting for cluster=\"cluster-zstream-4-22-phvfxj\" in resourcegroup=\"rg-zstream-upgrade-4-22-g9lt82\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-phvfxj in resource group rg-zstream-upgrade-4-22-g9lt82, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded"}},{"row_id":"cec986d133a05ca8b2ebf43f925468127ed1aec5ea85c201c3ba04989ec5c95a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","occurred_at":"2026-06-09T02:00:03Z","failed_at":"e2e","signature_id":"ca41a13db418271cbeff80f8bede1b8af93828c5394f8d69b3f67de6c998632d","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137]: failed to create HCP cluster \"cluster-zstream-4-19-dv6mdg\" with version 4.19.31 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000bd9ad0\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-dv6mdg, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-dv6mdg in resource group rg-zstream-upgrade-4-19-brhmk9, error: failed waiting for cluster=\"cluster-zstream-4-19-dv6mdg\" in resourcegroup=\"rg-zstream-upgrade-4-19-brhmk9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-dv6mdg in resource group rg-zstream-upgrade-4-19-brhmk9, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137]: failed to create HCP cluster \"cluster-zstream-4-19-dv6mdg\" with version 4.19.31 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000bd9ad0\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-dv6mdg, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-dv6mdg in resource group rg-zstream-upgrade-4-19-brhmk9, error: failed waiting for cluster=\"cluster-zstream-4-19-dv6mdg\" in resourcegroup=\"rg-zstream-upgrade-4-19-brhmk9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-dv6mdg in resource group rg-zstream-upgrade-4-19-brhmk9, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded"}},{"row_id":"d549b9ee0af7fde5b0fc27f1f2827de9cd56e57285e3a68c053900dbea513292","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","occurred_at":"2026-06-09T02:00:03Z","failed_at":"e2e","signature_id":"4d175bb2e3a50c064fd19145dad25d9bc38d60a209a424b50532bf368e56fea0","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a18200\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-9hkgrv\" resourcegroup=\"rg-np-downgrade-9hkgrv-4rg65k\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-9hkgrv-4rg65k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-9hkgrv/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": invalid node pool version 4.21.17: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": cannot downgrade from version 4.21.18 to 4.21.17\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:619]: failed to update nodepool npdg-4-21 to downgrade target 4.21.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a18200\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-9hkgrv\" resourcegroup=\"rg-np-downgrade-9hkgrv-4rg65k\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-9hkgrv-4rg65k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-9hkgrv/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"2 errors occurred\",\n        \"details\": [\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": invalid node pool version 4.21.17: cannot downgrade from current version 4.21.18\",\n            \"target\": \"properties.version.id\"\n          },\n          {\n            \"code\": \"InvalidRequestContent\",\n            \"message\": \"Invalid value: \\\"4.21.17\\\": cannot downgrade from version 4.21.18 to 4.21.17\",\n            \"target\": \"properties.version.id\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d81a996cdf791decc51216fb7be72b9db7c9b98c36c458e851bda1b4a4b600a6","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InvalidRequestContent; detail message Invalid value: \"4.21.17\": cannot downgrade from version 4.21.18 to 4.21.17; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"e78e6a23bf0c78d05a772f20aece30593be58c64bac9a1255b8ad3796588ced6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","occurred_at":"2026-06-09T02:00:03Z","failed_at":"e2e","signature_id":"86519bea76cee7f46f7264e61e683906b531a249469acbbb7fa8b6400176ea5d","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000af0900\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-brgdg7\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group complex-cilium-kv-brgdg7, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000af0900\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-brgdg7\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group complex-cilium-kv-brgdg7, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded"}},{"row_id":"fc1221f5c2c435b5de7cbf2a40cdf1e4e7cf04a04b39bc82f9497bc6d3c88d02","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2064165596991328256","occurred_at":"2026-06-09T02:00:03Z","failed_at":"e2e","signature_id":"f3e57f3875d065bdf55d7525d1966201997ac0ac1a15a5e3a3d5bceeae51438f","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137]: failed to create HCP cluster \"cluster-zstream-4-21-2bd9v7\" with version 4.21.17 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00173e930\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-2bd9v7, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-2bd9v7 in resource group rg-zstream-upgrade-4-21-g2d2z5, error: failed waiting for cluster=\"cluster-zstream-4-21-2bd9v7\" in resourcegroup=\"rg-zstream-upgrade-4-21-g2d2z5\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-2bd9v7 in resource group rg-zstream-upgrade-4-21-g2d2z5, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137]: failed to create HCP cluster \"cluster-zstream-4-21-2bd9v7\" with version 4.21.17 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00173e930\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-2bd9v7, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-2bd9v7 in resource group rg-zstream-upgrade-4-21-g2d2z5, error: failed waiting for cluster=\"cluster-zstream-4-21-2bd9v7\" in resourcegroup=\"rg-zstream-upgrade-4-21-g2d2z5\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-2bd9v7 in resource group rg-zstream-upgrade-4-21-g2d2z5, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-xmtssq, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: failed waiting for cluster=\"cluster-candidate-4-21-xmtssq\" in resourcegroup=\"rg-candidate-4-21-7g8djb\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-xmtssq in resource group rg-candidate-4-21-7g8djb, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","53fa6318769bb2a786556b3fac950c44b85ab8ba4f6fa60b2949a759bcc916c1","d81a996cdf791decc51216fb7be72b9db7c9b98c36c458e851bda1b4a4b600a6"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}