{"meta":{"date":"2026-04-07","timezone":"UTC","generated_at":"2026-10-31T22:13:39Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":82,"failed_runs":60,"runs_with_occurrences":60,"runs_with_failure_pattern_matches":59,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":22,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3960/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041666159912161280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3960,"pr_state":"closed","pr_sha":"b65c3a633c23914dd7243b1a0245bb0be24e2387","final_merged_sha":"b65c3a633c23914dd7243b1a0245bb0be24e2387","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T23:55:19Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b14c6c6f4e70de0ced5f33c29eacf87e0d7b04c6d9dc68f400bb4dce81e748fc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3960/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041666159912161280","occurred_at":"2026-04-07T23:55:19Z","failed_at":"e2e","signature_id":"d6908eadcccbbeecd3b4002995f38698988e0bddfc56c81dabca8dae07f9092d","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0013beb70\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n774fc, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-n774fc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n774fc, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n774fc, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-n774fc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n774fc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0016d1db0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n774fc\",\n },\n \u003c*fmt.wrapErrors | 0xc0013beb40\u003e{\n msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-n774fc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n774fc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0016d1db0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n774fc\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0013beb70\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n774fc, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-n774fc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n774fc, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n774fc, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-n774fc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n774fc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0016d1db0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n774fc\",\n },\n \u003c*fmt.wrapErrors | 0xc0013beb40\u003e{\n msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-n774fc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n774fc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0016d1db0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-n774fc\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-5sjrl6\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-5sjrl6, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041665858652082176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4770,"pr_state":"closed","pr_sha":"0469cc10771411b52260544ac6b5a788cad9d043","final_merged_sha":"0469cc10771411b52260544ac6b5a788cad9d043","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T23:54:07Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"086526876d5c4b691ab234725eb49fe1613417810d45785e714f112bca8c0091","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041665858652082176","occurred_at":"2026-04-07T23:54:07Z","failed_at":"e2e","signature_id":"2879bb8684c84b76d8825b61a4e5f63370c731e6a35e95ad89095cff65074bef","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:281]: Timed out after 30.001s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc001065d60\u003e: \n read access verification failed: get pods -n kube-system: pods is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"pods\" in API group \"\" in the namespace \"kube-system\"\n get configmaps -n kube-system: configmaps is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"configmaps\" in API group \"\" in the namespace \"kube-system\"\n {\n msg: \"read access verification failed: get pods -n kube-system: pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\\nget configmaps -n kube-system: configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.joinError | 0xc0011802e8\u003e{\n errs: [\n \u003c*fmt.wrapError | 0xc001065a80\u003e{\n msg: \"get pods -n kube-system: pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.StatusError | 0xc001184460\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n Reason: \"Forbidden\",\n Details: {Name: \"\", Group: \"\", Kind: \"pods\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n },\n },\n \u003c*fmt.wrapError | 0xc001065d00\u003e{\n msg: \"get configmaps -n kube-system: configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.StatusError | 0xc0011846e0\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n Reason: \"Forbidden\",\n Details: {Name: \"\", Group: \"\", Kind: \"configmaps\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n },\n },\n ],\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:281]: Timed out after 30.001s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc001065d60\u003e: \n read access verification failed: get pods -n kube-system: pods is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"pods\" in API group \"\" in the namespace \"kube-system\"\n get configmaps -n kube-system: configmaps is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"configmaps\" in API group \"\" in the namespace \"kube-system\"\n {\n msg: \"read access verification failed: get pods -n kube-system: pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\\nget configmaps -n kube-system: configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.joinError | 0xc0011802e8\u003e{\n errs: [\n \u003c*fmt.wrapError | 0xc001065a80\u003e{\n msg: \"get pods -n kube-system: pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.StatusError | 0xc001184460\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n Reason: \"Forbidden\",\n Details: {Name: \"\", Group: \"\", Kind: \"pods\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n },\n },\n \u003c*fmt.wrapError | 0xc001065d00\u003e{\n msg: \"get configmaps -n kube-system: configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.StatusError | 0xc0011846e0\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n Reason: \"Forbidden\",\n Details: {Name: \"\", Group: \"\", Kind: \"configmaps\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n },\n },\n ],\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3c6a69913a4e7e3ddde30c4ad0c3578c4319954e53163360fc7432ced31dda4a","failure_pattern":"read access verification failed: get pod \u003cpod\u003e kube-system: pods is forbidden: User \"system:sre-break-glass:\u003cuuid\u003e\" cannot list resource \"pods\" in API group \"\" in the namespace \"\u003cnamespace\u003e\"","search_query":"read access verification failed: get pods -n kube-system: pods is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"pods\" in API group \"\" in the namespace \"kube-system\""}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["3c6a69913a4e7e3ddde30c4ad0c3578c4319954e53163360fc7432ced31dda4a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041665655211560960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4775,"pr_state":"closed","pr_sha":"3eda0629846526464fbcb6bae8350a6247ae006b","final_merged_sha":"8cab26644417156b4fba886229fd5dd09cf35db9","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-04-07T23:53:19Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3973/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041658910158360576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3973,"pr_state":"closed","pr_sha":"445d308e2fc4c51ba6b9b76df86f35addde8a91b","final_merged_sha":"445d308e2fc4c51ba6b9b76df86f35addde8a91b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T23:26:30Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"0f4c400d5f7e507f55f8cef27102fb35f423c0629ca1e54f2b9150515c285b69","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3973/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041658910158360576","occurred_at":"2026-04-07T23:26:30Z","failed_at":"e2e","signature_id":"f6d629a93137c90d1f3007d2310746cc518ed7eecf27f426613f29d88a6c4a07","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00164c4e0\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-5tmmtt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5tmmtt, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-update-nodes\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-5tmmtt\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5tmmtt, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000502e90\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5tmmtt\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00164c4e0\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-5tmmtt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5tmmtt, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-update-nodes\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-5tmmtt\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5tmmtt, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000502e90\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-5tmmtt\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-82qzvg\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-82qzvg, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041658237983395840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3973,"pr_state":"closed","pr_sha":"445d308e2fc4c51ba6b9b76df86f35addde8a91b","final_merged_sha":"445d308e2fc4c51ba6b9b76df86f35addde8a91b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T23:23:50Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"cb14ace063b4f968069f749ca55ab58be329c72b3c2e076d20a056aae59d5949","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041658237983395840","occurred_at":"2026-04-07T23:23:50Z","failed_at":"e2e","signature_id":"a3ae2d087518e94823318a27f5fd49c3ee0f086087a1750ccf3cf1e335bc7129","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:178]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000778ab0\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-mfgc49\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-mfgc49, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-update-nodes\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-mfgc49\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-mfgc49, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010453c0\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-mfgc49\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:178]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000778ab0\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-mfgc49\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-mfgc49, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-update-nodes\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-mfgc49\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-mfgc49, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010453c0\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-mfgc49\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-82qzvg\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-82qzvg, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4549/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041650127017349120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4549,"pr_state":"closed","pr_sha":"8ac51d708827eb5c10759a0f8669b57677375e65","final_merged_sha":"8ac51d708827eb5c10759a0f8669b57677375e65","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-07T22:51:36Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041647150286245888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4640,"pr_state":"closed","pr_sha":"545b3a19237659318fe3e335269b1b9eb5cbe5e7","final_merged_sha":"e23d7c6b5233dccc23b0d4b24f1d727a143ec398","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T22:39:47Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f6c3e2d823ad9c965a5eb8bea51b21c9a678681ffb48415a21f543fc36ead160","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041647150286245888","occurred_at":"2026-04-07T22:39:47Z","failed_at":"e2e","signature_id":"0f14a9f032befd00df9256fdf268b4f1c650f1868bee4b78b7e29818b64c3ba0","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001402138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listk8zfxb\" in resourcegroup=\"ea-list-z8ctx9\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000839d60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listk8zfxb\\\" in resourcegroup=\\\"ea-list-z8ctx9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000839d40\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listk8zfxb\\\" in resourcegroup=\\\"ea-list-z8ctx9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000839d20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listk8zfxb\\\" in resourcegroup=\\\"ea-list-z8ctx9\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xbee3c60\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001402138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listk8zfxb\" in resourcegroup=\"ea-list-z8ctx9\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000839d60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listk8zfxb\\\" in resourcegroup=\\\"ea-list-z8ctx9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000839d40\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listk8zfxb\\\" in resourcegroup=\\\"ea-list-z8ctx9\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000839d20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listk8zfxb\\\" in resourcegroup=\\\"ea-list-z8ctx9\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xbee3c60\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-5v4nrk\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4610/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041628248961454080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4610,"pr_state":"open","pr_sha":"4252e835d95c998b32f62a787cc5c502e0b09c14","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T21:24:40Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"948330f7e35fec71497dbd3f16d43eec484424f999694f205990f7796fb51cb5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4610/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041628248961454080","occurred_at":"2026-04-07T21:24:40Z","failed_at":"e2e","signature_id":"418bf6148e63237cdbd7bb255231ee790c6a02af8bf570910f121655dff2b71f","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009ccba0\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-dp4vkt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-dp4vkt, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-dp4vkt\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-dp4vkt, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ef7460\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-dp4vkt\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009ccba0\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-dp4vkt\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-dp4vkt, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-dp4vkt\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-dp4vkt, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ef7460\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-dp4vkt\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-82qzvg\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-82qzvg, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4741/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041626469368598528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4741,"pr_state":"closed","pr_sha":"80a5f12065160304bbd1d53f7b889648d9535260","final_merged_sha":"80a5f12065160304bbd1d53f7b889648d9535260","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T21:17:33Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"17d18046e03707cef6331fb2954667fcd475a5c3094c551a342b0b84048a8e25","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4741/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041626469368598528","occurred_at":"2026-04-07T21:17:33Z","failed_at":"e2e","signature_id":"8bf4fd2ffe08633753cb5ed04e505455808d67a21f60302e8906840b9f8f0aaf","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0008f8870\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-jwcldw\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-jwcldw, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-jwcldw\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-jwcldw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00095e7d0\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-jwcldw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0008f8870\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-jwcldw\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-jwcldw, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-jwcldw\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-jwcldw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00095e7d0\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-jwcldw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-82qzvg\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-82qzvg, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3960/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041626467686682624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3960,"pr_state":"closed","pr_sha":"b65c3a633c23914dd7243b1a0245bb0be24e2387","final_merged_sha":"b65c3a633c23914dd7243b1a0245bb0be24e2387","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T21:17:32Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d6c99f6ad5dbdb0837d152bd6c60058adf135c118157d37c629d7938e9be9786","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3960/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041626467686682624","occurred_at":"2026-04-07T21:17:32Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-04-07T21:44:35.555Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-04-07T21:44:37.375Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-04-07T21:44:38.602Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=53e79c6547c190102e361853e32914aa8f9d94a6ffc4551d2694bdb94b1357e5\ntime=2026-04-07T21:46:09.141Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j6682624/providers/Microsoft.Resources/deployments/53e79c6547c190102e361853e32914aa8f9d94a6ffc4551d2694bdb94b1357e5/operationStatuses/08584260086076059012\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6682624/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6682624/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j6682624\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 6990b488-2629-4452-b874-8c5f69382fc5:4/7/2026 9:45:45 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4742/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041626456974430208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4742,"pr_state":"closed","pr_sha":"0bf0a618a05556febf3b9222e320b1eee2870842","final_merged_sha":"1bfd250498d2614c6267b766c8617d348665ea23","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-04-07T21:17:29Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4541/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041626421826162688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4541,"pr_state":"closed","pr_sha":"364fb5c63fca1e0cbf62bd6496b150c1c7d401dd","final_merged_sha":"364fb5c63fca1e0cbf62bd6496b150c1c7d401dd","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T21:17:25Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c22b4c09c917e8318dde09a6548a06a38b782a9cd267b98ff50d2ca673f9ba04","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4541/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041626421826162688","occurred_at":"2026-04-07T21:17:25Z","failed_at":"e2e","signature_id":"5ef8d7ea3ea1dca8ae8dd434c7a8508d0f5bbb7688db4b8601bad9d5a2ee7724","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00097b5f0\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-zh5dl7\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-zh5dl7, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-zh5dl7\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-zh5dl7, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00180b200\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-zh5dl7\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00097b5f0\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-zh5dl7\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-zh5dl7, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-zh5dl7\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-zh5dl7, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00180b200\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-zh5dl7\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-82qzvg\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-82qzvg, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4764/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041626400426823680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4764,"pr_state":"closed","pr_sha":"37a97d6c878754d1f0e530e8914d4b27bd0c532c","final_merged_sha":"37a97d6c878754d1f0e530e8914d4b27bd0c532c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T21:17:20Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"11aac0640312a744b582307a0ef5332fe2a908ec90db1d31e140e76cfd02855a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4764/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041626400426823680","occurred_at":"2026-04-07T21:17:20Z","failed_at":"e2e","signature_id":"a511fce12a4b92c62a6b5517f86830ecbaf29d68e8c8954e309a0520fabb1b91","test_name":"Customer should be able to list HCP clusters without node pools at the resource group level","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246]: Unexpected error:\n \u003c*errors.joinError | 0xc0013d05d0\u003e: \n failed to cleanup resource group: failed to detach NSGs from subnets in resource group cluster-listing-mrkzdx: failed listing vnets in resource group cluster-listing-mrkzdx: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-mrkzdx/providers/Microsoft.Network/virtualNetworks\n --------------------------------------------------------------------------------\n RESPONSE 429: 429 Too Many Requests\n ERROR CODE: ResourceCollectionRequestsThrottled\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceCollectionRequestsThrottled\",\n \"message\": \"Operation 'Microsoft.Network/virtualNetworks/read' failed as server encountered too many requests. Please try after '68' seconds. Tracking Id is '688e3dd6-f2c9-4b75-a5b3-d795903d7449'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246]: Unexpected error:\n \u003c*errors.joinError | 0xc0013d05d0\u003e: \n failed to cleanup resource group: failed to detach NSGs from subnets in resource group cluster-listing-mrkzdx: failed listing vnets in resource group cluster-listing-mrkzdx: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-mrkzdx/providers/Microsoft.Network/virtualNetworks\n --------------------------------------------------------------------------------\n RESPONSE 429: 429 Too Many Requests\n ERROR CODE: ResourceCollectionRequestsThrottled\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceCollectionRequestsThrottled\",\n \"message\": \"Operation 'Microsoft.Network/virtualNetworks/read' failed as server encountered too many requests. Please try after '68' seconds. Tracking Id is '688e3dd6-f2c9-4b75-a5b3-d795903d7449'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"27391593c89e1b6a3153b058ef31be166dd830240fb93e3caa733a62c02edc05","failure_pattern":"ERROR CODE: ResourceCollectionRequestsThrottled; provider Microsoft.Network","search_query":"ERROR CODE: ResourceCollectionRequestsThrottled"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["27391593c89e1b6a3153b058ef31be166dd830240fb93e3caa733a62c02edc05"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041626394219253760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4601,"pr_state":"closed","pr_sha":"eec30d3130530d4de45de55d7a0842d3ecaf04cc","final_merged_sha":"d4656bcc056e17f94a38d7d26dc82809bf007acc","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T21:17:18Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"29341dbd583825179e27edc695365a6c604a229d6674f95d12935589beb47e7c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4601/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041626394219253760","occurred_at":"2026-04-07T21:17:18Z","failed_at":"provision","signature_id":"e2a2d1bf4ed579663d9b017c82249065e232b94e20ea09082a623b40a048d3bc","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/aro-hcp-exporter","test_suite":"step graph","failure_text":"time=2026-04-07T22:03:49.184Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter description=\"Step aro-hcp-exporter\\n Kind: Helm\\n\"\ntime=2026-04-07T22:03:49.704Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter values=\"map[cacheTTL:60m clusterNames:prow-j9253760-svc,prow-j9253760-mgmt-1 collectionInterval:2m deployment:map[name:aro-hcp-exporter] enabledCollectors:service-tag-usage,kusto-logs-current image:map[digest:sha256:f51a27f9b992cc21fee6f455dce01c0b50341ca82c03a52332467a10e226ca9e registry:arohcpsvcdev.azurecr.io repository:arohcpmetrics] kustRegion:East US 2 kustoCluster:hcp-dev-us-2 listenAddress::8080 resources:map[limits:map[memory:unlimited] requests:map[cpu:100m memory:128Mi]] service:map[name:aro-hcp-exporter] serviceAccount:map[create:true msiClientId:0996400d-ab85-4da6-8a01-785ddc7ec814 name:aro-hcp-exporter] subscriptionNames:[XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX]]\"\ntime=2026-04-07T22:03:49.704Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-04-07T22:03:49.969Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter namespace=aro-hcp-exporter\ntime=2026-04-07T22:03:50.226Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter dryRun=false\ntime=2026-04-07T22:03:50.226Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-04-07T22:03:50.345Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-04-07T22:08:53.301Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter err=\"resource not ready, name: aro-hcp-exporter, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-04-07T22:08:53.301Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-04-07T22:08:53.301Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-04-07T22:08:55.495Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter release=aro-hcp-exporter namespace=aro-hcp-exporter status=failed description=\"Release \\\"aro-hcp-exporter\\\" failed: resource not ready, name: aro-hcp-exporter, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[cacheTTL:60m clusterNames:prow-j9253760-svc,prow-j9253760-mgmt-1 collectionInterval:2m deployment:map[name:aro-hcp-exporter] enabledCollectors:service-tag-usage,kusto-logs-current image:map[digest:sha256:f51a27f9b992cc21fee6f455dce01c0b50341ca82c03a52332467a10e226ca9e registry:arohcpsvcdev.azurecr.io repository:arohcpmetrics] kustRegion:East US 2 kustoCluster:hcp-dev-us-2 listenAddress::8080 resources:map[limits:map[memory:unlimited] requests:map[cpu:100m memory:128Mi]] service:map[name:aro-hcp-exporter] serviceAccount:map[create:true msiClientId:0996400d-ab85-4da6-8a01-785ddc7ec814 name:aro-hcp-exporter] subscriptionNames:[XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX]]\"\ntime=2026-04-07T22:08:55.496Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5yRwU7rMBBF9/mKUVaNlFRR+vpAQV0gwQrBBlZFLBz7qnWbeCx72lKpH49CBS2olYDt8Z1re07SQigg8ipoRJqQUaJENS0G3Cyg5c46U0cJ1s1y2qMH1eETYQ33TqJX+gNnzwkRUfqIsLYa11rzykmaU6oCF3PtC7x6DoJwmn2Z/u3YDXzL2w5/vvCenRU+kzzBkperZLlqEBwE8bZfSEx2tJkjgMR2iKI6Tw1kAzgaGCXo8SCtyup/Uf4ryounqqrLUT0up2lGwyGdzVzW49E0zbJkRwu2jpbWmYl1DuFIIzs62DvW9t1XsiMfuD89PDQnz2Yf/llJTi3PcuoQo5r1lRwMAjXbo88bRP0WAAD//7voq5RsAgAA\"\ntime=2026-04-07T22:08:55.498Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter\ntime=2026-04-07T22:08:55.499Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=aro-hcp-exporter err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-exporter, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6dbeb3937b77256442d5e74610d29f4e60a9d50cd5d1acd06f96957071a87d25","failure_pattern":"resource not ready, name: aro-hcp-exporter, kind: Deployment, status: InProgress context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-exporter, kind: Deployment, status: InProgress\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6dbeb3937b77256442d5e74610d29f4e60a9d50cd5d1acd06f96957071a87d25"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041626394676432896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4651,"pr_state":"closed","pr_sha":"6d025920a926606b0cf9684cc84a473cb8e69ee0","final_merged_sha":"6d025920a926606b0cf9684cc84a473cb8e69ee0","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-07T21:17:18Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4771/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041625605362946048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4771,"pr_state":"closed","pr_sha":"5b35760875077d9d675849752d78ac6e35863050","final_merged_sha":"5b35760875077d9d675849752d78ac6e35863050","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T21:14:10Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"52231d2dc36708e40d4f151058a61743acb3b0f51207a9e547af220438ac0be4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4771/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041625605362946048","occurred_at":"2026-04-07T21:14:10Z","failed_at":"e2e","signature_id":"f930fa8a1aa7377536e5528b1e1938e60581453d4246361d3aa619ea7d2bcfbc","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:363]: Timed out after 300.000s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:348 with:\nkubectl should execute successfully from authorized VM with OIDC token\nUnexpected error:\n \u003c*errors.errorString | 0xc001738e80\u003e: \n E0407 23:10:13.008148 2565 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n {\n s: \"E0407 23:10:13.008148 2565 memcache.go:265] \\\"Unhandled Error\\\" err=\\\"couldn't get current server API group list: the server has asked for the client to provide credentials\\\"\",\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:363]: Timed out after 300.000s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:348 with:\nkubectl should execute successfully from authorized VM with OIDC token\nUnexpected error:\n \u003c*errors.errorString | 0xc001738e80\u003e: \n E0407 23:10:13.008148 2565 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\"\n {\n s: \"E0407 23:10:13.008148 2565 memcache.go:265] \\\"Unhandled Error\\\" err=\\\"couldn't get current server API group list: the server has asked for the client to provide credentials\\\"\",\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"784380ace55b355f47065469da48bc1c7b73547d879efb7edb5ed96de913234e","failure_pattern":"couldn't get current server API group list: the server has asked for the client to provide credentials","search_query":"E0407 23:10:13.008148 2565 memcache.go:265] \"Unhandled Error\" err=\"couldn't get current server API group list: the server has asked for the client to provide credentials\""}},{"row_id":"5d51b3e66ca98211445a69ebbf4024e4b6b81a8e51a2e6418634023db0026549","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4771/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041625605362946048","occurred_at":"2026-04-07T21:14:10Z","failed_at":"e2e","signature_id":"b507fa262edda861cae0fa88df35de561783d2fd1b5a26ef3db6ad4b478f2706","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000da01b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-19-ngrddg\" in resourcegroup=\"rg-zstream-upgrade-4-19-s57gkp\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001293580\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-19-ngrddg\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-s57gkp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001293560\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-19-ngrddg\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-s57gkp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001293540\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-ngrddg\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-s57gkp\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xbee3c60\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000da01b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-19-ngrddg\" in resourcegroup=\"rg-zstream-upgrade-4-19-s57gkp\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001293580\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-19-ngrddg\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-s57gkp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001293560\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cluster-zstream-4-19-ngrddg\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-s57gkp\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc001293540\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-ngrddg\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-s57gkp\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xbee3c60\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-5v4nrk\" to finish deleting: context canceled"}},{"row_id":"703927110a8e1e42a3cdad184e99c6542cce3b7cb9c52f6f465d404e79e9f762","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4771/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041625605362946048","occurred_at":"2026-04-07T21:14:10Z","failed_at":"e2e","signature_id":"c503eea33bd121b2b7b757ab43d2e8dd21bfe3d5dc040446cc71b1981c163e75","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:210]: Timed out after 600.008s.\nboth ImageDigestMirrorSet entries should exist on the hosted cluster\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000748310\u003e: \n IDMS verification failed:\n source \"fake-source2.example.com/fake\" not found in any IDMS\n {\n s: \"IDMS verification failed:\\nsource \\\"fake-source2.example.com/fake\\\" not found in any IDMS\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:210]: Timed out after 600.008s.\nboth ImageDigestMirrorSet entries should exist on the hosted cluster\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000748310\u003e: \n IDMS verification failed:\n source \"fake-source2.example.com/fake\" not found in any IDMS\n {\n s: \"IDMS verification failed:\\nsource \\\"fake-source2.example.com/fake\\\" not found in any IDMS\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7dbb3245ba2adfe36d40679785a616da1431c27d6cab66a4f5200c0ae786eac3","failure_pattern":"source \"fake-source2.example.com/fake\" not found in any IDMS","search_query":"source \"fake-source2.example.com/fake\" not found in any IDMS"}},{"row_id":"79ac41f1e2870851e26b605a9d6abed100ac628e3f3236320d05ca5e122f9154","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4771/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041625605362946048","occurred_at":"2026-04-07T21:14:10Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"d23843e70ab506edf76db321e52c9e4eb990ed99ed70267b6a8704f85000bb39","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4771/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041625605362946048","occurred_at":"2026-04-07T21:14:10Z","failed_at":"e2e","signature_id":"58d9e1a879dff30004308b8a40bdaa98823b4d0008157d05dff06d16cf74f6af","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00190a168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-5v4nrk\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d01b60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-5v4nrk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d01b40\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-5v4nrk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d01b20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-5v4nrk\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xbee3c60\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc00190a168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-5v4nrk\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d01b60\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-5v4nrk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d01b40\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-5v4nrk\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d01b20\u003e{\n msg: \"failed waiting for hcpCluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-5v4nrk\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xbee3c60\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-5v4nrk\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","784380ace55b355f47065469da48bc1c7b73547d879efb7edb5ed96de913234e","7dbb3245ba2adfe36d40679785a616da1431c27d6cab66a4f5200c0ae786eac3"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4695/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041623989259866112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4695,"pr_state":"closed","pr_sha":"164ac19981c0c71407ac65088ceb1c824aef2b97","final_merged_sha":"3a0b8ba847a3b9f4913be118e29015fc3ad907bf","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-04-07T21:07:45Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041621950081536000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4752,"pr_state":"closed","pr_sha":"e93d6fcbbcbd1f7b713bc37a3e878ac6010e5b65","final_merged_sha":"a288ef3c8fe9167b54a5e847c9e16150a9f04cd2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T20:59:38Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b122613488db16e47664ede5627c3fbb522887f84b43b08e63566011f115f371","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041621950081536000","occurred_at":"2026-04-07T20:59:38Z","failed_at":"provision","signature_id":"0cfdbc69e36cd9409140a1d94f3eca54c19e2976278c5f5c45b1651fb5c59fcf","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster","test_suite":"step graph","failure_text":"time=2026-04-07T21:20:04.663Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-04-07T21:20:05.466Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster\ntime=2026-04-07T21:20:06.760Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster deployment=be70dc36ca8a5932419e13113a877d2b9c8ff775020a84f4bca80e22bfde8c20\ntime=2026-04-07T21:31:39.467Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j1536000-mgmt-1/providers/Microsoft.Resources/deployments/be70dc36ca8a5932419e13113a877d2b9c8ff775020a84f4bca80e22bfde8c20/operationStatuses/08584260100794067375\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1536000-mgmt-1/providers/Microsoft.Resources/deployments/maestro-consumer\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1536000-mgmt-1/providers/Microsoft.Resources/deployments/maestro-eg-crt-gj4osxq4swds2\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1536000-mgmt-1/providers/Microsoft.Resources/deployments/maestro-eg-crt-gj4osxq4swds2\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1536000-mgmt-1/providers/Microsoft.Resources/deployments/hcp-underlay-j1536000-mgmt-1-cert\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1536000-mgmt-1/providers/Microsoft.Resources/deployments/hcp-underlay-j1536000-mgmt-1-cert\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j1536000-mgmt-1/providers/Microsoft.Resources/deploymentScripts/newCert-hcp-underlay-j1536000-mgmt-1\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentScriptOperationFailed\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"A task was canceled.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e816f1e3b47b563efe8a6cd99a62f5abdf0ac3394314f730c55c2f66121c5a89","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentScriptOperationFailed; detail message A task was canceled.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e816f1e3b47b563efe8a6cd99a62f5abdf0ac3394314f730c55c2f66121c5a89"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4786/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041620052268027904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4786,"pr_state":"closed","pr_sha":"c3141a0ada4a8dc07a320c23e351a2c9956aa34d","final_merged_sha":"8617904be0ed03c40d37fc170ee79321f3e1366b","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T20:52:06Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"2272dd5e20ea67930e13b0ba94f349f572d954af6ddb96e0bd3839313be81736","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4786/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041620052268027904","occurred_at":"2026-04-07T20:52:06Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4730/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041616171807019008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4730,"pr_state":"closed","pr_sha":"3f5edbc77a965d57ce9fc10be9ea9e5d6c520fa1","final_merged_sha":"c9f07cdaf77afc3c4d41fed584593740f5f63fcc","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-04-07T20:36:41Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041615350503575552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3973,"pr_state":"closed","pr_sha":"445d308e2fc4c51ba6b9b76df86f35addde8a91b","final_merged_sha":"445d308e2fc4c51ba6b9b76df86f35addde8a91b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T20:33:25Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"50d132346914519624e0ca610d2a95db74db70b2103bb8ddb16bac4eb9f25020","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041615350503575552","occurred_at":"2026-04-07T20:33:25Z","failed_at":"e2e","signature_id":"86cfb077bf7087a007fb130475425c974cf00dcd843ded0d2dcb6df55e23042b","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010fd050\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-flqgvs\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-flqgvs, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-update-nodes\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-flqgvs\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-flqgvs, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009624a0\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-flqgvs\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010fd050\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-flqgvs\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-flqgvs, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-update-nodes\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-flqgvs\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-flqgvs, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009624a0\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-flqgvs\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-82qzvg\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-82qzvg, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4768/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041595017331478528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4768,"pr_state":"closed","pr_sha":"5bd37521bbeb13c152bba83f6ab9686da749a046","final_merged_sha":"5bd37521bbeb13c152bba83f6ab9686da749a046","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-07T19:12:36Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4784/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041584328521027584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4784,"pr_state":"closed","pr_sha":"d37f428b114681531d592baecb0969df6ed372c5","final_merged_sha":"d37f428b114681531d592baecb0969df6ed372c5","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-07T18:30:09Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4785/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041579738161483776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4785,"pr_state":"closed","pr_sha":"8da3f0ab8b61109576a170a60df3694beeedf801","final_merged_sha":"aa6bb1c70f5c0830e5677d664e10c86bf40fdf6a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T18:11:54Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"32cb13c489dd469b131cd9f59e0f58d70ad9358c142fe09ff735ff7957eaebda","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4785/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041579738161483776","occurred_at":"2026-04-07T18:11:54Z","failed_at":"e2e","signature_id":"efcf21d487a440a3c8a7884f35637799bebbb4a0c336174789fbe495ef98bb87","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:375]: Unexpected error:\n \u003ccontext.deadlineExceededError\u003e: \n context deadline exceeded\n {}\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:375]: Unexpected error:\n \u003ccontext.deadlineExceededError\u003e: \n context deadline exceeded\n {}\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2136fa2e6655fbc68bffdf51efbecc1a8fa59a5f574143c87b971c52c50880cb","failure_pattern":"context deadline exceeded","search_query":"context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["2136fa2e6655fbc68bffdf51efbecc1a8fa59a5f574143c87b971c52c50880cb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4765/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041574146223116288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4765,"pr_state":"closed","pr_sha":"2fef8f741791bafc091714f2971c6292e78f2885","final_merged_sha":"2fef8f741791bafc091714f2971c6292e78f2885","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T17:49:41Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"00395e1a42bd4a5e0e7f31519e5f16c37deabedd9fc91b281110c996f6316ff9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4765/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041574146223116288","occurred_at":"2026-04-07T17:49:41Z","failed_at":"e2e","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:35]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:35]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4518/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041569968193015808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4518,"pr_state":"closed","pr_sha":"bb1f0486021536b45ce14b6f7c222429e11dd51a","final_merged_sha":"c68c3104b7e5c84dff8c2653a50758b036821419","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-04-07T17:33:05Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4772/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041565642099789824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4772,"pr_state":"closed","pr_sha":"14c315a53cc8ef7296ac80ba8f1749759c564d48","final_merged_sha":"14c315a53cc8ef7296ac80ba8f1749759c564d48","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-07T17:15:54Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4767/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041559452485685248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4767,"pr_state":"closed","pr_sha":"61ffd5deff7e7bbe5fc002326577754c895ea305","final_merged_sha":"61ffd5deff7e7bbe5fc002326577754c895ea305","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-07T16:51:18Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4768/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041558776502292480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4768,"pr_state":"closed","pr_sha":"5bd37521bbeb13c152bba83f6ab9686da749a046","final_merged_sha":"5bd37521bbeb13c152bba83f6ab9686da749a046","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-07T16:48:33Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4783/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041550715247136768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4783,"pr_state":"closed","pr_sha":"ffc4d3b13af395f5cbb6008af57187800a46344a","final_merged_sha":"74a99a673a093eaa041c6289fe7fdfc65a792e3f","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-04-07T16:15:39Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4518/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041549352165445632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4518,"pr_state":"closed","pr_sha":"bb1f0486021536b45ce14b6f7c222429e11dd51a","final_merged_sha":"c68c3104b7e5c84dff8c2653a50758b036821419","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T16:11:02Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"edc6d5486654602d4dc203f95c03405fc71cc1d63ecddaf6689d1f665630fe28","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4518/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041549352165445632","occurred_at":"2026-04-07T16:11:02Z","failed_at":"provision","signature_id":"136a3db43cde2d2a9c8c0ee0008498f1341add56ea32a74b765a8511e2b1111e","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.HypershiftOperator/management/deploy","test_suite":"step graph","failure_text":"time=2026-04-07T16:55:20.117Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.HypershiftOperator resourceGroup=management step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-04-07T16:55:20.598Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.HypershiftOperator resourceGroup=management step=deploy values=\"map[additionalArgs:--enable-cpo-overrides apiVersion:v1 azureKeyVaultClientId:05509e00-6630-42f1-9676-1d8e379855f9 description:A Helm chart to manage Hypershift Operator and dependencies for ARO HCP image:arohcpsvcdev.azurecr.io/redhat-services-prod/crt-redhat-acm-tenant/hypershift/hypershift-operator imageDigest:sha256:c5787a7f15979007174b4099b9f91e449c797c78aff00eb0787a510e4c7cb73a limitClusterSizes:true name:aro-hcp-hypershift-operator operatorEnvVars:map[sharedIngressIPTag:] registryOverrides:quay.io/openshift-release-dev/ocp-v4.0-art-dev=arohcpocpdev.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev,quay.io/openshift-release-dev/ocp-release=arohcpocpdev.azurecr.io/openshift-release-dev/ocp-release,registry.redhat.io/redhat=arohcpocpdev.azurecr.io/redhat version:0.1.0]\"\ntime=2026-04-07T16:55:20.598Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.HypershiftOperator resourceGroup=management step=deploy\ntime=2026-04-07T16:55:20.770Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.HypershiftOperator resourceGroup=management step=deploy namespace=hypershift\ntime=2026-04-07T16:55:21.028Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.HypershiftOperator resourceGroup=management step=deploy dryRun=false\ntime=2026-04-07T16:55:21.028Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.HypershiftOperator resourceGroup=management step=deploy\ntime=2026-04-07T16:55:21.141Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.HypershiftOperator resourceGroup=management step=deploy\ntime=2026-04-07T17:00:46.363Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.HypershiftOperator resourceGroup=management step=deploy err=\"failed post-install: resource still exists, name: cluster, kind: ClusterSizingConfiguration, status: Current\\ncontext deadline exceeded\"\ntime=2026-04-07T17:00:46.363Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.HypershiftOperator resourceGroup=management step=deploy\ntime=2026-04-07T17:00:46.364Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.HypershiftOperator resourceGroup=management step=deploy\ntime=2026-04-07T17:00:47.742Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.HypershiftOperator resourceGroup=management step=deploy release=hypershift namespace=hypershift status=failed description=\"Release \\\"hypershift\\\" failed: failed post-install: resource still exists, name: cluster, kind: ClusterSizingConfiguration, status: Current\\ncontext deadline exceeded\" values=\"map[additionalArgs:--enable-cpo-overrides apiVersion:v1 azureKeyVaultClientId:05509e00-6630-42f1-9676-1d8e379855f9 description:A Helm chart to manage Hypershift Operator and dependencies for ARO HCP image:arohcpsvcdev.azurecr.io/redhat-services-prod/crt-redhat-acm-tenant/hypershift/hypershift-operator imageDigest:sha256:c5787a7f15979007174b4099b9f91e449c797c78aff00eb0787a510e4c7cb73a limitClusterSizes:true name:aro-hcp-hypershift-operator operatorEnvVars:map[sharedIngressIPTag:] registryOverrides:quay.io/openshift-release-dev/ocp-v4.0-art-dev=arohcpocpdev.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev,quay.io/openshift-release-dev/ocp-release=arohcpocpdev.azurecr.io/openshift-release-dev/ocp-release,registry.redhat.io/redhat=arohcpocpdev.azurecr.io/redhat version:0.1.0]\"\ntime=2026-04-07T17:00:47.742Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.HypershiftOperator resourceGroup=management step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/4yRy2rrMBCG936KQasY7OCEXMCHLM45dFXoou0qpQtZ+usokSUzkhMCefjihFx6gy716ZuR5p/EIhIj+I4VAi1IyyijrCwGvlpDxXvjdBkiG1dndEIPssEFYQt3JKGV6ozTl4SISDyBt0bhr1K+c1FkJFb7FhxW5i3mxoUorQV/5CI71yrGsabtrM3D5fhV/W+7EMGP3uKfcdq4+qencnVSc/YWvSSS1z/JpqvADhHhrp8mJAfarcCgaBqEKJuWKsQd4GigZUSPB2JcjGd5McmL+fNoVk6n5Xi0FCkNh/S9My+LopzMliJNkwOtvXG0MU4vjHPgmx14R9fobzP/HHZyoJZ9f3v9aEat1yf5d00ysr7OqEEIsu5betZgqvY3w2sE9R4AAP//jWR47ikCAAA=\"\ntime=2026-04-07T17:00:47.743Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.HypershiftOperator resourceGroup=management step=deploy\ntime=2026-04-07T17:00:47.743Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.HypershiftOperator resourceGroup=management step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed post-install: resource still exists, name: cluster, kind: ClusterSizingConfiguration, status: Current\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8a67fb18cde59e982d261fa14da04adbe57a8089da27870eacd61c5b49db33a5","failure_pattern":"failed post-install: resource still exists, name: cluster, kind: ClusterSizingConfiguration, status: Current context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed post-install: resource still exists, name: cluster, kind: ClusterSizingConfiguration, status: Current\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8a67fb18cde59e982d261fa14da04adbe57a8089da27870eacd61c5b49db33a5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4738/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041544371207147520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4738,"pr_state":"closed","pr_sha":"c0686000dfb7e6c66b1b27dc139ca5419e6326a7","final_merged_sha":"c0686000dfb7e6c66b1b27dc139ca5419e6326a7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T15:51:22Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"491103d29278cf27166f14920bf207427516c2cf32515b752e943251783b48fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4738/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041544371207147520","occurred_at":"2026-04-07T15:51:22Z","failed_at":"provision","signature_id":"6fc2cab87d2187f33f0596f0dbc728e7a00de4b46e6bd1e2759a55f7d788395a","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/service/deploy","test_suite":"step graph","failure_text":"time=2026-04-07T16:12:36.816Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-04-07T16:12:37.540Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:e2a2ae33-d91d-4d37-ad98-db4bd18fd5a3 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j7147520] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j7147520.hcpsvc.osadev.cloud] image:map[digest:sha256:9f97f4c7686d52025c8d7f26d84f670440e6d1657fcca2aa7028120cc9968e4f registry:arohcpsvcdev.azurecr.io repository:ci-op-l55nkg4n/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-l55nkg4n/pipeline:pull workloadIdentityClientId:cd463ced-dc7c-45f6-a28e-48a2d2fd732c workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:0765675b-2d26-453b-a02e-85b189bbd431 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-04-07T16:12:37.540Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-04-07T16:12:37.890Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy namespace=sessiongate\ntime=2026-04-07T16:12:38.184Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy dryRun=false\ntime=2026-04-07T16:12:38.184Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-04-07T16:12:38.300Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-04-07T16:12:45.287Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF\"\ntime=2026-04-07T16:12:45.287Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-04-07T16:12:45.287Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-04-07T16:12:47.491Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy release=sessiongate namespace=sessiongate status=failed description=\"Release \\\"sessiongate\\\" failed: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF\" values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:e2a2ae33-d91d-4d37-ad98-db4bd18fd5a3 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j7147520] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j7147520.hcpsvc.osadev.cloud] image:map[digest:sha256:9f97f4c7686d52025c8d7f26d84f670440e6d1657fcca2aa7028120cc9968e4f registry:arohcpsvcdev.azurecr.io repository:ci-op-l55nkg4n/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-l55nkg4n/pipeline:pull workloadIdentityClientId:cd463ced-dc7c-45f6-a28e-48a2d2fd732c workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:0765675b-2d26-453b-a02e-85b189bbd431 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-04-07T16:12:47.496Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/6RUwW4aMRC971dYewIJrwhN04oqB5pUqlSpQoRTqh683mFx8NrbmTGIiI+vzEJDNyEQ5bb7/OZ5/OZpEgssEMgH1EDiWhSKFavcQsfnD6D5h3HFkBiNK3uigX6qCv5BsAS3RahWeg93fyVCCJHeAS6NhpHWPjhOeyIlIDLelYrh2W9Tc2MDMeDE2zZD7r5lpZwqoYJGcl8YK74aVxhXtgu1d4zeWkCZv8h4m4YFVQBKsKA59nNCE/4EIB4FnoNjo1WsaatjQ5Iq8PyIzh1oBB6jX5oC8MYqomcG6TpCakHSEBsvjSsRIm/nbSD21WQ37luYGWda3VB2IJgp9HKu60w9BoRM+6pt+HGXXnH4fGuPWrHN1XmBuoXa+vU+Le/gSw3IZhYnGAc2Q6B5886jho80joO1B5Gqg7UnAhOT4tE8boMy9tbo9fYSa/2qfc3bBSpgNJoknXbi+3Q6nvjwwvn/v8nvL8ki5IAOGOhb3AiUbMRqDgiCTQXEqqpFDrwCcKJTKIYId9JBf3Al+5ey/2l6cTW8GAw/fL5PuyLLxCucy4/3abebbMSDN04sjCuujXOAB3vMO/G0vg73VnthJRtRo4+nT432RO2LhnyeSE9YX/ZEBUSqjJIeC0CRrw8eXwDpvwEAAP//9PxjU20FAAA=\"\ntime=2026-04-07T16:12:47.498Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-04-07T16:12:47.499Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d8ac2e6cca70c7690a0f75cd38b523f3cb17701626a2f823018ba74b9a5ecb16","failure_pattern":"Internal error occurred: failed calling webhook \"validation.istio.io\": failed to call webhook: Post \"\u003curl\u003e\": EOF","search_query":"error running Helm release deployment Step, failed to deploy helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d8ac2e6cca70c7690a0f75cd38b523f3cb17701626a2f823018ba74b9a5ecb16"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4765/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041543568786460672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4765,"pr_state":"closed","pr_sha":"2fef8f741791bafc091714f2971c6292e78f2885","final_merged_sha":"2fef8f741791bafc091714f2971c6292e78f2885","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T15:47:51Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"f8fd7fa459c435ec359d495a7d12b881c4ac5b510584f95634b8075023fdef10","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4765/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041543568786460672","occurred_at":"2026-04-07T15:47:51Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-04-07T15:49:38.994Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-04-07T15:49:41.565Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-04-07T15:49:42.869Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=f7d533cc7b9129215361140abb57e9a43693e9c7b7f193833ff288c72864d34b\ntime=2026-04-07T15:51:43.541Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j6460672/providers/Microsoft.Resources/deployments/f7d533cc7b9129215361140abb57e9a43693e9c7b7f193833ff288c72864d34b/operationStatuses/08584260299034408648\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6460672/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6460672/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041538562238713856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4599,"pr_state":"closed","pr_sha":"177cea95835e607c5f3514b939390be80f97193b","final_merged_sha":"f6ed2d92c3031e65888f81d9423e89db816cc046","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T15:28:17Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"0e5c78cda440405f00427c9dfa4898dd958d20c4280aae1185fa5997f5b97315","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041538562238713856","occurred_at":"2026-04-07T15:28:17Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-04-07T15:50:21.773Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-04-07T15:50:23.576Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-04-07T15:50:24.936Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=4faeb0d0adbcc607b29f9fd0c0baeb35b55d27521d6d4416fb0248ca2fefb15f\ntime=2026-04-07T15:52:25.525Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j8713856/providers/Microsoft.Resources/deployments/4faeb0d0adbcc607b29f9fd0c0baeb35b55d27521d6d4416fb0248ca2fefb15f/operationStatuses/08584260298611986025\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8713856/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8713856/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4541/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041534194806427648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4541,"pr_state":"closed","pr_sha":"364fb5c63fca1e0cbf62bd6496b150c1c7d401dd","final_merged_sha":"364fb5c63fca1e0cbf62bd6496b150c1c7d401dd","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T15:10:56Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d4d5f9741a6a1f1d0bd775fee4fa5c64552511122ec0bda67d0aa000fd56341e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4541/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041534194806427648","occurred_at":"2026-04-07T15:10:56Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-04-07T15:36:54.406Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-04-07T15:36:56.297Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-04-07T15:36:57.503Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=c8d915714aec58c3d784d5422b61f51aa9d69f9c03106e4d04db9b2c99878b5a\ntime=2026-04-07T15:38:28.109Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j6427648/providers/Microsoft.Resources/deployments/c8d915714aec58c3d784d5422b61f51aa9d69f9c03106e4d04db9b2c99878b5a/operationStatuses/08584260306686758966\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6427648/providers/Microsoft.Resources/deployments/svc-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6427648/providers/Microsoft.Resources/deployments/hcp-monitor\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4738/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041529187717615616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4738,"pr_state":"closed","pr_sha":"c0686000dfb7e6c66b1b27dc139ca5419e6326a7","final_merged_sha":"c0686000dfb7e6c66b1b27dc139ca5419e6326a7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T14:48:59Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"6dd15b05a74c885bc0f3ce34490cee0c5eb06bf0007fc54f102fc812ebf75abc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4738/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041529187717615616","occurred_at":"2026-04-07T14:48:59Z","failed_at":"provision","signature_id":"0cfdbc69e36cd9409140a1d94f3eca54c19e2976278c5f5c45b1651fb5c59fcf","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster","test_suite":"step graph","failure_text":"time=2026-04-07T15:23:34.070Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-04-07T15:23:34.764Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster\ntime=2026-04-07T15:23:36.262Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster deployment=ff5bf241e0199a807fde04d9682c2e13d55095ee1d611cfbc9c65a3d5dc32cfd\ntime=2026-04-07T15:36:09.208Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7615616-mgmt-1/providers/Microsoft.Resources/deployments/ff5bf241e0199a807fde04d9682c2e13d55095ee1d611cfbc9c65a3d5dc32cfd/operationStatuses/08584260314699748756\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7615616-mgmt-1/providers/Microsoft.Resources/deployments/3d87b52a-4ffb-5597-a771-884937a644a5\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7615616-mgmt-1/providers/Microsoft.Resources/deployments/604c1139-1539-56c5-9dec-39a42a581815\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7615616-mgmt-1/providers/Microsoft.Resources/deployments/3e87119d-65de-50c7-8cc9-2c2950e05b90\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7615616-mgmt-1/providers/Microsoft.Resources/deployments/0186cdd9-543f-5ac4-8cf6-adb977c06c7c\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7615616-mgmt-1/providers/Microsoft.Resources/deployments/metrics-infra\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n },\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7615616-mgmt-1/providers/Microsoft.Resources/deployments/00dd06c9-975a-5a4e-bef3-6912148edd09\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7615616-mgmt-1/providers/Microsoft.Resources/deployments/7e088ec6-6d16-5064-bdb7-40a982b1a3aa\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"RoleAssignmentLimitExceeded\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"No more role assignments can be created.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["852f049f2b2782678be16e8a6f33ad96db70ab779fbda534649aef9586a6f8bb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041524599060959232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4640,"pr_state":"closed","pr_sha":"e444319a3349dd4f909c6d25564f6a75db38b8ad","final_merged_sha":"e23d7c6b5233dccc23b0d4b24f1d727a143ec398","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-04-07T14:32:38Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041524290670563328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4004,"pr_state":"closed","pr_sha":"ff5b88c96712835bead8c845c79f61dfb754f685","final_merged_sha":"9feee09ab6996cfe7dd433b262345799b7851593","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-04-07T14:31:25Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4769/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041522104284745728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4769,"pr_state":"closed","pr_sha":"2b482983a796bfd9f7f0239ba127db29c9afda93","final_merged_sha":"2b482983a796bfd9f7f0239ba127db29c9afda93","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-07T14:22:53Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041522074089951232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4752,"pr_state":"closed","pr_sha":"1f0df3d37327c808442ecde151888a7306d17201","final_merged_sha":"a288ef3c8fe9167b54a5e847c9e16150a9f04cd2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T14:22:46Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"52b249a2bec99e933f50182f300eef2ed08ade2f5537af94d59b4bc61e1328a0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041522074089951232","occurred_at":"2026-04-07T14:22:46Z","failed_at":"provision","signature_id":"5e6291fad96b6e69174d9796359ecd54fd14e68feddce0e2bc70918ac445ce20","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-04-07T15:03:01.230Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image description=\"Step mirror-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e sessiongate.image.registry \u003cnil\u003e}:{\u003cnil\u003e sessiongate.image.repository \u003cnil\u003e}@{\u003cnil\u003e sessiongate.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-04-07T15:05:25.932Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build05.ci.openshift.org registry.build05.ci.openshift.org registry.build05.ci.openshift.org registry.build05.ci.openshift.org\\nSetting up registry authentication for CI source registry.\\ninfo: Using registry public hostname registry.build05.ci.openshift.org\\nSaved credentials for registry.build05.ci.openshift.org into /tmp/tmp.d0LjBtveX3/containers/auth.json\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.build05.ci.openshift.org/ci-op-t2t4c3qx/pipeline@sha256:32bca2525ca68099de49d98b5c5f2934e4298d3b48adc2fb8fd9c43943df86db to arohcpsvcdev.azurecr.io/ci-op-t2t4c3qx/pipeline:32bca2525ca68099de49d98b5c5f2934e4298d3b48adc2fb8fd9c43943df86db.\\nThe image will still be available under it's original digest sha256:32bca2525ca68099de49d98b5c5f2934e4298d3b48adc2fb8fd9c43943df86db in the target registry.\\nError: Put \\\"https://arohcpsvcdev.azurecr.io/v2/ci-op-t2t4c3qx/pipeline/blobs/uploads/6ba06b13-877d-49b3-bfa5-a5100603ceb4?_nouploadcache=false\u0026_state=Axl7D4u2aRjUf8igXVv5KE3BIidYuTWMggiD5pY9Ked7Ik5hbWUiOiJjaS1vcC10MnQ0YzNxeC9waXBlbGluZSIsIlVVSUQiOiI2YmEwNmIxMy04NzdkLTQ5YjMtYmZhNS1hNTEwMDYwM2NlYjQiLCJPZmZzZXQiOjAsIlN0YXJ0ZWRBdCI6IjIwMjYtMDQtMDdUMTU6MDM6MDguNjg5MDQ2MzdaIn0%3D\u0026digest=sha256%3Ac0b9efa017a51c19aa4fd15a8c5ac98f0ba0d91bb3c4b3bb55f60f22d400724f\\\": read tcp 172.24.34.71:40544-\u003e172.64.66.1:443: read: connection reset by peer\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7fed4d4df4591c7e3fe9f07aebddd940980fc09d73dd627d1834eabdc96bd811","failure_pattern":"Error: Put \"\u003curl\u003e\": read tcp \u003cip\u003e-\u003e\u003cip\u003e: read: connection reset by peer","search_query":"Error: Put \\\"https://arohcpsvcdev.azurecr.io/v2/ci-op-t2t4c3qx/pipeline/blobs/uploads/6ba06b13-877d-49b3-bfa5-a5100603ceb4?_nouploadcache=false\u0026_state=Axl7D4u2aRjUf8igXVv5KE3BIidYuTWMggiD5pY9Ked7Ik5hbWUiOiJjaS1vcC10MnQ0YzNxeC9waXBlbGluZSIsIlVVSUQiOiI2YmEwNmIxMy04NzdkLTQ5YjMtYmZhNS1hNTEwMDYwM2NlYjQiLCJPZmZzZXQiOjAsIlN0YXJ0ZWRBdCI6IjIwMjYtMDQtMDdUMTU6MDM6MDguNjg5MDQ2MzdaIn0%3D\u0026digest=sha256%3Ac0b9efa017a51c19aa4fd15a8c5ac98f0ba0d91bb3c4b3bb55f60f22d400724f\\\": read tcp 172.24.34.71:40544-\u003e172.64.66.1:443: read: connection reset by peer"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7fed4d4df4591c7e3fe9f07aebddd940980fc09d73dd627d1834eabdc96bd811"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4740/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041520616057606144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4740,"pr_state":"closed","pr_sha":"0231f5fe5b294adf2b0e7846420ef7a9580ba370","final_merged_sha":"389466a234f4716ad49f8b021a30f1f018b06ea6","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T14:16:59Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1ca8d0d629946bf1b7a19acefc458d8eff5f5e010b414cf45956d6f2664ad357","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4740/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041520616057606144","occurred_at":"2026-04-07T14:16:59Z","failed_at":"e2e","signature_id":"8a6707eae7fc3e71b3797aea9684e962c4f28e9bca8eff0391e3d646df85c247","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000962660\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-q87kfm\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-q87kfm, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-q87kfm\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-q87kfm, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010713a0\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-q87kfm\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000962660\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-q87kfm\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-q87kfm, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-q87kfm\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-q87kfm, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010713a0\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-q87kfm\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-82qzvg\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-82qzvg, error: context deadline exceeded"}},{"row_id":"1ce431504d212e0433d9b896c88c0c53d8ec493de55e9908ef0e205a43ca17be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4740/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041520616057606144","occurred_at":"2026-04-07T14:16:59Z","failed_at":"e2e","signature_id":"a2c041c74ae89f79d86418fb0a6967634e7dcda92a802281e5797e6f8f1fe587","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:252]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e3f350\u003e: \n add password after 4 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n {\n msg: \"add password after 4 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\",\n errs: [\n \u003c*odataerrors.ODataError | 0xc000e8fda0\u003e{\n ApiError: {\n Message: \"\",\n ResponseStatusCode: 403,\n ResponseHeaders: {\n header: {\n headers: {\n \"vary\": {\n \"Accept-Encoding\": {},\n },\n \"x-ms-ags-diagnostic\": {\n \"{\\\"ServerInfo\\\":{\\\"DataCenter\\\":\\\"Canada Central\\\",\\\"Slice\\\":\\\"E\\\",\\\"Ring\\\":\\\"3\\\",\\\"ScaleUnit\\\":\\\"000\\\",\\\"RoleInstance\\\":\\\"TO1PEPF0002409A\\\"}}\": {},\n },\n \"x-ms-resource-unit\": {\"1\": {}},\n \"date\": {\n \"Tue, 07 Apr 2026 15:30:16 GMT\": {},\n },\n \"strict-transport-security\": {\n \"max-age=31536000\": {},\n },\n \"request-id\": {\n \"b8aa8af6-04b2-4f29-ae63-a8e8b5764dfb\": {},\n },\n \"content-type\": {\n \"application/json\": {},\n },\n \"client-request-id\": {\n \"b8aa8af6-04b2-4f29-ae63-a8e8b5764dfb\": {},\n },\n \"cache-control\": {\"no-cache\": {}},\n },\n },\n },\n },\n backingStore: \u003c*store.InMemoryBackingStore | 0xc000a6c7a0\u003e{\n returnOnlyChangedValues: false,\n initializationCompleted: false,\n store: {\n \"errorEscaped\": \u003c*odataerrors.MainError | 0xc00140e620\u003e{\n backingStore: \u003c*store.InMemoryBackingStore | 0xc000a6c7c0\u003e{\n returnOnlyChangedValues: false,\n initializationCompleted: false,\n store: {\n \"innerError\": \u003c*odataerrors.InnerError | 0xc00140e680\u003e{\n backingStore: \u003c*store.InMemoryBackingStore | 0xc000a6c7e0\u003e{\n returnOnlyChangedValues: ...,\n initializationCompleted: ...,\n store: ...,\n subscribers: ...,\n changedValues: ...,\n },\n },\n \"code\": \u003c*string | 0xc00140e490\u003e\"Authorization_RequestDenied\",\n \"message\": \u003c*string | 0xc00140e4e0\u003e\"Insufficient privileges to complete the operation.\",\n },\n subscribers: {},\n changedValues: {\"code\": false, \"message\": false, \"innerError\": false},\n },\n },\n },\n subscribers: {},\n changedValues: {\n \"errorEscaped\": false,\n },\n },\n },\n \u003c*odataerrors.ODataError | 0xc000e8fda0\u003e{\n ApiError: {\n Message: \"\",\n ResponseStatusCode: 403,\n ResponseHeaders: {\n header: {\n headers: {\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:252]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e3f350\u003e: \n add password after 4 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\n {\n msg: \"add password after 4 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API\",\n errs: [\n \u003c*odataerrors.ODataError | 0xc000e8fda0\u003e{\n ApiError: {\n Message: \"\",\n ResponseStatusCode: 403,\n ResponseHeaders: {\n header: {\n headers: {\n \"vary\": {\n \"Accept-Encoding\": {},\n },\n \"x-ms-ags-diagnostic\": {\n \"{\\\"ServerInfo\\\":{\\\"DataCenter\\\":\\\"Canada Central\\\",\\\"Slice\\\":\\\"E\\\",\\\"Ring\\\":\\\"3\\\",\\\"ScaleUnit\\\":\\\"000\\\",\\\"RoleInstance\\\":\\\"TO1PEPF0002409A\\\"}}\": {},\n },\n \"x-ms-resource-unit\": {\"1\": {}},\n \"date\": {\n \"Tue, 07 Apr 2026 15:30:16 GMT\": {},\n },\n \"strict-transport-security\": {\n \"max-age=31536000\": {},\n },\n \"request-id\": {\n \"b8aa8af6-04b2-4f29-ae63-a8e8b5764dfb\": {},\n },\n \"content-type\": {\n \"application/json\": {},\n },\n \"client-request-id\": {\n \"b8aa8af6-04b2-4f29-ae63-a8e8b5764dfb\": {},\n },\n \"cache-control\": {\"no-cache\": {}},\n },\n },\n },\n },\n backingStore: \u003c*store.InMemoryBackingStore | 0xc000a6c7a0\u003e{\n returnOnlyChangedValues: false,\n initializationCompleted: false,\n store: {\n \"errorEscaped\": \u003c*odataerrors.MainError | 0xc00140e620\u003e{\n backingStore: \u003c*store.InMemoryBackingStore | 0xc000a6c7c0\u003e{\n returnOnlyChangedValues: false,\n initializationCompleted: false,\n store: {\n \"innerError\": \u003c*odataerrors.InnerError | 0xc00140e680\u003e{\n backingStore: \u003c*store.InMemoryBackingStore | 0xc000a6c7e0\u003e{\n returnOnlyChangedValues: ...,\n initializationCompleted: ...,\n store: ...,\n subscribers: ...,\n changedValues: ...,\n },\n },\n \"code\": \u003c*string | 0xc00140e490\u003e\"Authorization_RequestDenied\",\n \"message\": \u003c*string | 0xc00140e4e0\u003e\"Insufficient privileges to complete the operation.\",\n },\n subscribers: {},\n changedValues: {\"code\": false, \"message\": false, \"innerError\": false},\n },\n },\n },\n subscribers: {},\n changedValues: {\n \"errorEscaped\": false,\n },\n },\n },\n \u003c*odataerrors.ODataError | 0xc000e8fda0\u003e{\n ApiError: {\n Message: \"\",\n ResponseStatusCode: 403,\n ResponseHeaders: {\n header: {\n headers: {\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5700a7b5ac643e41503dee012c26f2c5ae2fb0b9e206eab0a7840d3b0746911a","failure_pattern":"add password after 4 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API","search_query":"add password after 4 attempts; last attempt error: error status code received from the API; polling error: error status code received from the API"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","5700a7b5ac643e41503dee012c26f2c5ae2fb0b9e206eab0a7840d3b0746911a"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041519193290969088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4556,"pr_state":"open","pr_sha":"5522f5c5d772d72b7203db87283538da7f9406a1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T14:11:19Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"2185d9674c68d959d680b4d62b67df729c95feaef79ec98387b2faddc96e4ffb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041519193290969088","occurred_at":"2026-04-07T14:11:19Z","failed_at":"e2e","signature_id":"bf0b589591f3678806906f3888aa7d3230bede2dede2f38d237739d92ac22956","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:108]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f24330\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-z6gwpj\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-z6gwpj, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-z6gwpj\\\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-z6gwpj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012323f0\u003e{\n s: \"timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-z6gwpj\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:108]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f24330\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-z6gwpj\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-z6gwpj, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-z6gwpj\\\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-z6gwpj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012323f0\u003e{\n s: \"timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-z6gwpj\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-5sjrl6\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-5sjrl6, error: context deadline exceeded"}},{"row_id":"d294c00842f04a07a254a4593b5a3ddc7dd743b4ea48bc1dbd671edc891431e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4556/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041519193290969088","occurred_at":"2026-04-07T14:11:19Z","failed_at":"e2e","signature_id":"9721279f6a05c36f5e48df6d02d7cd64d94ec97b7f4c66b6ce3cff12eeb1ff2e","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:116]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e5e570\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-5sjrl6\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-5sjrl6, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-5sjrl6\\\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-5sjrl6, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011a2a60\u003e{\n s: \"timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-5sjrl6\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:116]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e5e570\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-5sjrl6\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-5sjrl6, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-5sjrl6\\\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-5sjrl6, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011a2a60\u003e{\n s: \"timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-5sjrl6\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-5sjrl6\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-5sjrl6, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4611/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041517631097278464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4611,"pr_state":"closed","pr_sha":"47e9dedcc13d2360ddc61db9458339fbe2281138","final_merged_sha":"47e9dedcc13d2360ddc61db9458339fbe2281138","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-07T14:05:06Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041517272798859264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4479,"pr_state":"closed","pr_sha":"eea15b6230f44952cf4bbd42bfa448513feedc2f","final_merged_sha":"6b3f6dbe2a33440b5ea52c19d436bc0f656e7289","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-04-07T14:03:41Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4781/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041516960067358720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4781,"pr_state":"closed","pr_sha":"723bdbb67ead25eb9ad23752bfb9bb1a90470dfc","final_merged_sha":"cdcee87d00c4a609142e1bc8ac955e149111757c","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-04-07T14:02:27Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4772/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041513434545131520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4772,"pr_state":"closed","pr_sha":"14c315a53cc8ef7296ac80ba8f1749759c564d48","final_merged_sha":"14c315a53cc8ef7296ac80ba8f1749759c564d48","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T13:48:26Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b855bf2c7780b852dfec9c2710271851dd2bd604496b8b589bdd730c7ae49ece","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4772/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041513434545131520","occurred_at":"2026-04-07T13:48:26Z","failed_at":"e2e","signature_id":"9c66c62c900b99a5a45b8d8eac988aa909634c365cfaeb4ad8e73b31c6d9f283","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0008a4ff0\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-hxxqlz\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hxxqlz, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-hxxqlz\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hxxqlz, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0004970d0\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hxxqlz\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0008a4ff0\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-hxxqlz\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hxxqlz, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-hxxqlz\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hxxqlz, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0004970d0\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-hxxqlz\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-82qzvg\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-82qzvg, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4782/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041512099858878464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4782,"pr_state":"closed","pr_sha":"de7caefa0433b7c45785bb4f0270d5e7974d78f5","final_merged_sha":"29196eb5fccd69b78e533458e88791de6b493e0f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T13:43:06Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"10dcb8f53b17144f8342c1294a795eabee702c681516f2881f4fe2c30a288e9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4782/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041512099858878464","occurred_at":"2026-04-07T13:43:06Z","failed_at":"e2e","signature_id":"31b23c83df0b03ce8c23c9a6ca2eec3a4ae3c37bbd0eaaa4fade9826deee0865","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0013758c0\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-cnrdkx\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-cnrdkx, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-update-nodes\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-cnrdkx\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-cnrdkx, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00079a870\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-cnrdkx\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0013758c0\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-cnrdkx\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-cnrdkx, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-update-nodes\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-cnrdkx\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-cnrdkx, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00079a870\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-cnrdkx\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-82qzvg\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-82qzvg, error: context deadline exceeded"}},{"row_id":"2222b2670d323b6ca63013853e7c96d1402eeb9d34d9fc0871ff6f15a667cfdc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4782/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041512099858878464","occurred_at":"2026-04-07T13:43:06Z","failed_at":"e2e","signature_id":"23825dc40bf8149f5a785b9a4d3720e166df47a8bc446dfecc2502c8f885221f","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:210]: Timed out after 600.038s.\nboth ImageDigestMirrorSet entries should exist on the hosted cluster\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000a74320\u003e: \n IDMS verification failed:\n source \"fake-source2.example.com/fake\" not found in any IDMS\n {\n s: \"IDMS verification failed:\\nsource \\\"fake-source2.example.com/fake\\\" not found in any IDMS\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:210]: Timed out after 600.038s.\nboth ImageDigestMirrorSet entries should exist on the hosted cluster\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000a74320\u003e: \n IDMS verification failed:\n source \"fake-source2.example.com/fake\" not found in any IDMS\n {\n s: \"IDMS verification failed:\\nsource \\\"fake-source2.example.com/fake\\\" not found in any IDMS\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7dbb3245ba2adfe36d40679785a616da1431c27d6cab66a4f5200c0ae786eac3","failure_pattern":"source \"fake-source2.example.com/fake\" not found in any IDMS","search_query":"source \"fake-source2.example.com/fake\" not found in any IDMS"}},{"row_id":"eb8654ecb3d5f294985d985aa5351840209e5752eea89c42bdfe509f1cee9c2e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4782/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041512099858878464","occurred_at":"2026-04-07T13:43:06Z","failed_at":"e2e","signature_id":"580eb9cad9e5faa47dc9a0c3d7d5ffecbbcdf8c22e96572121d6e6591951b761","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001580168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-7sftkr\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00080b960\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-7sftkr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00080b940\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-7sftkr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00080b920\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-7sftkr\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xbee2c60\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc001580168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-7sftkr\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc00080b960\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-7sftkr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00080b940\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-7sftkr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc00080b920\u003e{\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-7sftkr\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xbee2c60\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-5v4nrk\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","567db2d6abee3120288f4d2589aaf4e556f2738a143b684ca0ae72211d9ed761","7dbb3245ba2adfe36d40679785a616da1431c27d6cab66a4f5200c0ae786eac3"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041510671530594304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4582,"pr_state":"closed","pr_sha":"453429fc550d57d7915964340a777a91b19a1208","final_merged_sha":"453429fc550d57d7915964340a777a91b19a1208","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-07T13:37:23Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4769/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041508395256647680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4769,"pr_state":"closed","pr_sha":"2b482983a796bfd9f7f0239ba127db29c9afda93","final_merged_sha":"2b482983a796bfd9f7f0239ba127db29c9afda93","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T13:28:20Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"014b9124dca7b58c9b95942bd88608ebb0a82112cf0f95f7b30f7f61672b8ed0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4769/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041508395256647680","occurred_at":"2026-04-07T13:28:20Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-04-07T14:03:39.418Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-04-07T14:03:41.630Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-04-07T14:03:43.273Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=1dfc65b962c6c8661ede2fa57afd3f88db6acfbb2adc62d25fc0dd9b87f54089\ntime=2026-04-07T14:05:44.246Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j6647680/providers/Microsoft.Resources/deployments/1dfc65b962c6c8661ede2fa57afd3f88db6acfbb2adc62d25fc0dd9b87f54089/operationStatuses/08584260362631382465\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6647680/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6647680/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j6647680\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 722bd24a-b788-430d-94fd-693b69c01952:4/7/2026 2:04:59 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4768/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041507835778437120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4768,"pr_state":"closed","pr_sha":"5bd37521bbeb13c152bba83f6ab9686da749a046","final_merged_sha":"5bd37521bbeb13c152bba83f6ab9686da749a046","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T13:26:05Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d6d81f17965335e31e989cb8634b5022c27307bd3a0b6a822bf9b4bb6e5ae490","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4768/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041507835778437120","occurred_at":"2026-04-07T13:26:05Z","failed_at":"e2e","signature_id":"725d262748a723bdbb4b6c3dd37da5cf092a1d0d77a6be101bceea35f2e1b07f","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010a8690\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-qmztlm\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-qmztlm, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-update-nodes\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-qmztlm\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-qmztlm, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000902160\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-qmztlm\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010a8690\u003e: \n failed waiting for nodepool=\"np-update-nodes\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-qmztlm\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-qmztlm, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-update-nodes\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-qmztlm\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-qmztlm, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000902160\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-nodes in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-qmztlm\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-82qzvg\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-82qzvg, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4765/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041504213699137536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4765,"pr_state":"closed","pr_sha":"2fef8f741791bafc091714f2971c6292e78f2885","final_merged_sha":"2fef8f741791bafc091714f2971c6292e78f2885","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T13:11:48Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1b3a90781261d184fae555660df21536fea2df60930d3ee8ec314bbfd548a88e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4765/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041504213699137536","occurred_at":"2026-04-07T13:11:48Z","failed_at":"e2e","signature_id":"26b1c4b7dc2017e98ec72f32223058f9cc92fc9b49225259b073532d31bc8e20","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:206]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0008af830\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-qn9mnj\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-qn9mnj, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-one-node\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-qn9mnj\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-qn9mnj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a80a00\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-qn9mnj\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:206]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0008af830\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-qn9mnj\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-qn9mnj, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-one-node\\\" in cluster=\\\"np-update-nodes-hcp-cluster\\\" resourcegroup=\\\"nodepool-update-nodes-qn9mnj\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-qn9mnj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a80a00\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-qn9mnj\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-82qzvg\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-82qzvg, error: context deadline exceeded"}},{"row_id":"4fccb2292c49c80985bf1dc334f7c4c9dd69b87539a36a22e159cac757fce4f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4765/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041504213699137536","occurred_at":"2026-04-07T13:11:48Z","failed_at":"e2e","signature_id":"b0d39be32c484f54739f06a4f6e64f78a26a9737f8ccc1024731ef40c3da8fb2","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0013a1380\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-82qzvg\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-82qzvg, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-82qzvg\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-82qzvg, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0008fba10\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-82qzvg\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0013a1380\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-82qzvg\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-82qzvg, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-82qzvg\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-82qzvg, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0008fba10\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-82qzvg\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-82qzvg\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-82qzvg, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4767/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041504096988434432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4767,"pr_state":"closed","pr_sha":"61ffd5deff7e7bbe5fc002326577754c895ea305","final_merged_sha":"61ffd5deff7e7bbe5fc002326577754c895ea305","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-07T13:11:20Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4769/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041498150329389056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4769,"pr_state":"closed","pr_sha":"d8f5225b7decb507a676e4b4ef2f5d9d85ffcd8c","final_merged_sha":"2b482983a796bfd9f7f0239ba127db29c9afda93","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T12:47:42Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"690aae5c77079055b7a0498f879635b05f95a945f6bdfc44374feaa323b61710","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4769/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041498150329389056","occurred_at":"2026-04-07T12:47:42Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-04-07T13:08:14.679Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-04-07T13:08:16.885Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-04-07T13:08:18.139Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=fcf1c8db7997a1ac664a5566dd8192905ab5c32a0cf8b9fe57d981e9110fd13d\ntime=2026-04-07T13:09:48.781Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9389056/providers/Microsoft.Resources/deployments/fcf1c8db7997a1ac664a5566dd8192905ab5c32a0cf8b9fe57d981e9110fd13d/operationStatuses/08584260395881090823\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9389056/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9389056/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j9389056\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report c4cbae0c-35c1-442f-bac6-dda4c316d936:4/7/2026 1:09:22 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4780/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041496741999546368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4780,"pr_state":"closed","pr_sha":"48ee8e48dd68430bd7db42358757d1fbec12418a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T12:42:07Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"6691bedd2ef1f5233e4295e76778976ba6ea9144affbc63fd2d07121fee7bef2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4780/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041496741999546368","occurred_at":"2026-04-07T12:42:07Z","failed_at":"e2e","signature_id":"faf8b67aaec1ff915e51ab67b4d226b06a3ad8a176b14a7ea9d99c976bce1d4c","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000948e70\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-npkv6l-jcnt89, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-npkv6l\" in resourcegroup=\"rg-np-version-upgrade-npkv6l-jcnt89\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-npkv6l-jcnt89, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-npkv6l\\\" in resourcegroup=\\\"rg-np-version-upgrade-npkv6l-jcnt89\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009142e0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-npkv6l-jcnt89\",\n },\n \u003c*fmt.wrapError | 0xc0009b2820\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-npkv6l\\\" in resourcegroup=\\\"rg-np-version-upgrade-npkv6l-jcnt89\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000948e70\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-npkv6l-jcnt89, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-npkv6l\" in resourcegroup=\"rg-np-version-upgrade-npkv6l-jcnt89\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-npkv6l-jcnt89, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-npkv6l\\\" in resourcegroup=\\\"rg-np-version-upgrade-npkv6l-jcnt89\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009142e0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-npkv6l-jcnt89\",\n },\n \u003c*fmt.wrapError | 0xc0009b2820\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-npkv6l\\\" in resourcegroup=\\\"rg-np-version-upgrade-npkv6l-jcnt89\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4763/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041489681509519360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4763,"pr_state":"closed","pr_sha":"f4919c57513de108fa8bf035cbf7cb6ab8e26ea6","final_merged_sha":"5cb8e0820f193d7c23a369dbbb0a969d37e7ada3","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T12:14:03Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"8136a621266429a83896522ede867a5915d8f85ffb5399ba012be0a499780553","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4763/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041489681509519360","occurred_at":"2026-04-07T12:14:03Z","failed_at":"e2e","signature_id":"a9ed7af947dc28f92c6ad8d2e3e55084b76d371f12c968ceb19bc8cf453779bd","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:155]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e06db0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-j2hxmt-xxghfq, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-j2hxmt\" in resourcegroup=\"rg-np-version-upgrade-j2hxmt-xxghfq\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-j2hxmt-xxghfq, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-j2hxmt\\\" in resourcegroup=\\\"rg-np-version-upgrade-j2hxmt-xxghfq\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00124d6c0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-j2hxmt-xxghfq\",\n },\n \u003c*fmt.wrapError | 0xc0005a8720\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-j2hxmt\\\" in resourcegroup=\\\"rg-np-version-upgrade-j2hxmt-xxghfq\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:155]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e06db0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-j2hxmt-xxghfq, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-j2hxmt\" in resourcegroup=\"rg-np-version-upgrade-j2hxmt-xxghfq\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-j2hxmt-xxghfq, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-j2hxmt\\\" in resourcegroup=\\\"rg-np-version-upgrade-j2hxmt-xxghfq\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00124d6c0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-j2hxmt-xxghfq\",\n },\n \u003c*fmt.wrapError | 0xc0005a8720\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-j2hxmt\\\" in resourcegroup=\\\"rg-np-version-upgrade-j2hxmt-xxghfq\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}},{"row_id":"d80b485d6956ca9b33d3183df34fec67bf7aa21e91eb77663c9fb79505f38c12","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4763/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041489681509519360","occurred_at":"2026-04-07T12:14:03Z","failed_at":"e2e","signature_id":"dac0c98a8545e4c9e07e3c74fbb0597c9ea501dda2b07535ee915a0890e03efb","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:235]: Expected success, but got an error:\n \u003c*errors.errorString | 0xc0006c48c0\u003e: \n expected 3 ready nodes in node pool \"npupgrade-4-20\", found 4\n {\n s: \"expected 3 ready nodes in node pool \\\"npupgrade-4-20\\\", found 4\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:235]: Expected success, but got an error:\n \u003c*errors.errorString | 0xc0006c48c0\u003e: \n expected 3 ready nodes in node pool \"npupgrade-4-20\", found 4\n {\n s: \"expected 3 ready nodes in node pool \\\"npupgrade-4-20\\\", found 4\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"45d6eb2440b6253fe7a78d69d27664918ed1383d05554034e8401d46c818affb","failure_pattern":"expected 3 ready nodes in node pool \"\u003cnodepool\u003e\", found 4","search_query":"expected 3 ready nodes in node pool \"npupgrade-4-20\", found 4"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["45d6eb2440b6253fe7a78d69d27664918ed1383d05554034e8401d46c818affb","a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4210/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041483661718589440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4210,"pr_state":"closed","pr_sha":"0a139023a67aaddf3889e54a80f39039340b4374","final_merged_sha":"ef912c114f3891ce814ee96ef6545953f99c7540","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T11:50:08Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1f496a83da16d3d09dfc3feaed03061cf0f24950c9a73c5b6d88eed47bbbe7c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4210/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041483661718589440","occurred_at":"2026-04-07T11:50:08Z","failed_at":"e2e","signature_id":"136778c08a38065e5f141aed27b868eb505fb0a92b6c58365ed81580cd17a974","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009eb350\u003e: \n failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-spw5z8, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-spw5z8\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-spw5z8, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-spw5z8, error: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-spw5z8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-spw5z8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e74b50\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-spw5z8\",\n },\n \u003c*fmt.wrapErrors | 0xc0009eb320\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-spw5z8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-spw5z8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e74b50\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-spw5z8\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009eb350\u003e: \n failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-spw5z8, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-spw5z8\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-spw5z8, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-spw5z8, error: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-spw5z8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-spw5z8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e74b50\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-spw5z8\",\n },\n \u003c*fmt.wrapErrors | 0xc0009eb320\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-spw5z8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-spw5z8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e74b50\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-spw5z8\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-5sjrl6\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-5sjrl6, error: context deadline exceeded"}},{"row_id":"31339a3f237019bca73303f8c1cf0f2f6475efec6b57249758f63f8470cff79c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4210/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041483661718589440","occurred_at":"2026-04-07T11:50:08Z","failed_at":"e2e","signature_id":"d379e3f4337cfa1844562b37f1c1519fd682373587e905f3a606514b8645654f","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009c94a0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-qb4n2n-468fmn, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-qb4n2n\" in resourcegroup=\"rg-np-version-upgrade-qb4n2n-468fmn\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-qb4n2n-468fmn, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-qb4n2n\\\" in resourcegroup=\\\"rg-np-version-upgrade-qb4n2n-468fmn\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001635730\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-qb4n2n-468fmn\",\n },\n \u003c*fmt.wrapError | 0xc000d8abe0\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-qb4n2n\\\" in resourcegroup=\\\"rg-np-version-upgrade-qb4n2n-468fmn\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009c94a0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-qb4n2n-468fmn, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-qb4n2n\" in resourcegroup=\"rg-np-version-upgrade-qb4n2n-468fmn\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-qb4n2n-468fmn, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-qb4n2n\\\" in resourcegroup=\\\"rg-np-version-upgrade-qb4n2n-468fmn\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001635730\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-qb4n2n-468fmn\",\n },\n \u003c*fmt.wrapError | 0xc000d8abe0\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-qb4n2n\\\" in resourcegroup=\\\"rg-np-version-upgrade-qb4n2n-468fmn\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4742/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041475072790630400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4742,"pr_state":"closed","pr_sha":"0bf0a618a05556febf3b9222e320b1eee2870842","final_merged_sha":"1bfd250498d2614c6267b766c8617d348665ea23","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T11:16:00Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a0478d1c8d9e6c92edc42caa50d201e9c5bbeb542ae3d3acbb353d4cdfbbd79d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4742/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041475072790630400","occurred_at":"2026-04-07T11:16:00Z","failed_at":"e2e","signature_id":"5f18c3c27496aed3a3545a451659a19beb748880ee921ec2ddf46b7b6171a949","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001880de0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-mbchm6-xhjzgb, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-mbchm6\" in resourcegroup=\"rg-np-version-upgrade-mbchm6-xhjzgb\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-mbchm6-xhjzgb, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-mbchm6\\\" in resourcegroup=\\\"rg-np-version-upgrade-mbchm6-xhjzgb\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0017872e0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-mbchm6-xhjzgb\",\n },\n \u003c*fmt.wrapError | 0xc000ea2500\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-mbchm6\\\" in resourcegroup=\\\"rg-np-version-upgrade-mbchm6-xhjzgb\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001880de0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-mbchm6-xhjzgb, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-mbchm6\" in resourcegroup=\"rg-np-version-upgrade-mbchm6-xhjzgb\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-mbchm6-xhjzgb, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-mbchm6\\\" in resourcegroup=\\\"rg-np-version-upgrade-mbchm6-xhjzgb\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0017872e0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-mbchm6-xhjzgb\",\n },\n \u003c*fmt.wrapError | 0xc000ea2500\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-mbchm6\\\" in resourcegroup=\\\"rg-np-version-upgrade-mbchm6-xhjzgb\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4743/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041473484088938496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4743,"pr_state":"closed","pr_sha":"b4fc9c3708e2d124a7b9a62a2905e33018cb3360","final_merged_sha":"80cdcc675989f7dcc63a578af3e13b5bd81f6807","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T11:09:41Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"0d7bc848f433498b20bda3503ca06e07763478035ce0388fe2575b9b2abc697e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4743/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041473484088938496","occurred_at":"2026-04-07T11:09:41Z","failed_at":"e2e","signature_id":"7693972d98acf46442543c5602f7a1c2d4421948dbb61069d9f76142b401b0d1","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0013779b0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-bwnvfc-ldds9x, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-bwnvfc\" in resourcegroup=\"rg-np-version-upgrade-bwnvfc-ldds9x\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-bwnvfc-ldds9x, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-bwnvfc\\\" in resourcegroup=\\\"rg-np-version-upgrade-bwnvfc-ldds9x\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00144f4a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-bwnvfc-ldds9x\",\n },\n \u003c*fmt.wrapError | 0xc000f8a880\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-bwnvfc\\\" in resourcegroup=\\\"rg-np-version-upgrade-bwnvfc-ldds9x\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0013779b0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-bwnvfc-ldds9x, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-bwnvfc\" in resourcegroup=\"rg-np-version-upgrade-bwnvfc-ldds9x\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-bwnvfc-ldds9x, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-bwnvfc\\\" in resourcegroup=\\\"rg-np-version-upgrade-bwnvfc-ldds9x\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00144f4a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-bwnvfc-ldds9x\",\n },\n \u003c*fmt.wrapError | 0xc000f8a880\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-bwnvfc\\\" in resourcegroup=\\\"rg-np-version-upgrade-bwnvfc-ldds9x\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4717/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041471032384032768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4717,"pr_state":"closed","pr_sha":"a9667630ea966ea2708ffd16022663a4c14633b2","final_merged_sha":"26416847807a241acf58f8fd074b514f61424362","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T10:59:57Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a5f21eb2065c147ce9f984772de4cf685cd615ddbf9aa99c550bafe29a32a2b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4717/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041471032384032768","occurred_at":"2026-04-07T10:59:57Z","failed_at":"e2e","signature_id":"59815f5a7ebfa173706884e87b084e2507ef2606e0e0f398d8435a1f09535cd7","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001702ff0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-bkfp5n-2bl7wn, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-bkfp5n\" in resourcegroup=\"rg-np-version-upgrade-bkfp5n-2bl7wn\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-bkfp5n-2bl7wn, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-bkfp5n\\\" in resourcegroup=\\\"rg-np-version-upgrade-bkfp5n-2bl7wn\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001700300\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-bkfp5n-2bl7wn\",\n },\n \u003c*fmt.wrapError | 0xc0009fe740\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-bkfp5n\\\" in resourcegroup=\\\"rg-np-version-upgrade-bkfp5n-2bl7wn\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001702ff0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-bkfp5n-2bl7wn, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-bkfp5n\" in resourcegroup=\"rg-np-version-upgrade-bkfp5n-2bl7wn\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-bkfp5n-2bl7wn, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-bkfp5n\\\" in resourcegroup=\\\"rg-np-version-upgrade-bkfp5n-2bl7wn\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001700300\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-bkfp5n-2bl7wn\",\n },\n \u003c*fmt.wrapError | 0xc0009fe740\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-bkfp5n\\\" in resourcegroup=\\\"rg-np-version-upgrade-bkfp5n-2bl7wn\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4772/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041468368967438336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4772,"pr_state":"closed","pr_sha":"14c315a53cc8ef7296ac80ba8f1749759c564d48","final_merged_sha":"14c315a53cc8ef7296ac80ba8f1749759c564d48","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T10:49:16Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4f25e73b80d737ad9a682af643b5cf95cee830d8cf7069750cb7bf31c1b74341","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4772/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041468368967438336","occurred_at":"2026-04-07T10:49:16Z","failed_at":"e2e","signature_id":"e6406c7b57080d3148493bf62d7c20a60dd201545aa1e689ef3404f030a79eef","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000efca20\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-4kzd6z-knmhzf, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-4kzd6z\" in resourcegroup=\"rg-np-version-upgrade-4kzd6z-knmhzf\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-4kzd6z-knmhzf, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-4kzd6z\\\" in resourcegroup=\\\"rg-np-version-upgrade-4kzd6z-knmhzf\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001236050\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-4kzd6z-knmhzf\",\n },\n \u003c*fmt.wrapError | 0xc000efa100\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-4kzd6z\\\" in resourcegroup=\\\"rg-np-version-upgrade-4kzd6z-knmhzf\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000efca20\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-4kzd6z-knmhzf, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-4kzd6z\" in resourcegroup=\"rg-np-version-upgrade-4kzd6z-knmhzf\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-4kzd6z-knmhzf, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-4kzd6z\\\" in resourcegroup=\\\"rg-np-version-upgrade-4kzd6z-knmhzf\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001236050\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-4kzd6z-knmhzf\",\n },\n \u003c*fmt.wrapError | 0xc000efa100\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-4kzd6z\\\" in resourcegroup=\\\"rg-np-version-upgrade-4kzd6z-knmhzf\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041467710289743872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4479,"pr_state":"closed","pr_sha":"2bb2cc27db8b0133c5cadfc4d061257264d9e571","final_merged_sha":"6b3f6dbe2a33440b5ea52c19d436bc0f656e7289","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T10:46:45Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"6b5eff5bb8332caa7033ca9b9273dd196486407b2fba30a18afbaafb9faaa874","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041467710289743872","occurred_at":"2026-04-07T10:46:45Z","failed_at":"e2e","signature_id":"cf336c72a916df0ccb0ff82b78daf366952f8960d1a1a79b9b07332589ebc020","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000ff2b40\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-tkqp2w-f6t92d, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-tkqp2w\" in resourcegroup=\"rg-np-version-upgrade-tkqp2w-f6t92d\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-tkqp2w-f6t92d, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-tkqp2w\\\" in resourcegroup=\\\"rg-np-version-upgrade-tkqp2w-f6t92d\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0008794a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-tkqp2w-f6t92d\",\n },\n \u003c*fmt.wrapError | 0xc00119a6e0\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-tkqp2w\\\" in resourcegroup=\\\"rg-np-version-upgrade-tkqp2w-f6t92d\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000ff2b40\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-tkqp2w-f6t92d, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-tkqp2w\" in resourcegroup=\"rg-np-version-upgrade-tkqp2w-f6t92d\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-tkqp2w-f6t92d, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-tkqp2w\\\" in resourcegroup=\\\"rg-np-version-upgrade-tkqp2w-f6t92d\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0008794a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-tkqp2w-f6t92d\",\n },\n \u003c*fmt.wrapError | 0xc00119a6e0\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-tkqp2w\\\" in resourcegroup=\\\"rg-np-version-upgrade-tkqp2w-f6t92d\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}},{"row_id":"a585bc22613e35d6efdb43c631eaa4337d6a2436c8e74c71f41a8dfcb2c6d8ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4479/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041467710289743872","occurred_at":"2026-04-07T10:46:45Z","failed_at":"e2e","signature_id":"bafd877af233cbffa8b09bb5175f8ba09db6247535578f972d450d791e79770e","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e283c0\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-g5rk4g\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-g5rk4g, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-g5rk4g\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-g5rk4g, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e75560\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-g5rk4g\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e283c0\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-g5rk4g\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-g5rk4g, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-g5rk4g\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-g5rk4g, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e75560\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-g5rk4g\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-82qzvg\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-82qzvg, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4779/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041466448785707008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4779,"pr_state":"closed","pr_sha":"38a93d20c02dfc76a235a5e34559d440fa9b33cc","final_merged_sha":"38a93d20c02dfc76a235a5e34559d440fa9b33cc","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-07T10:41:36Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4778/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041462227680104448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4778,"pr_state":"closed","pr_sha":"30d72b498b586ae7bc3c08947ee455d48754f748","final_merged_sha":"30d72b498b586ae7bc3c08947ee455d48754f748","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T10:24:58Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"330407f062dcf2ecf299dda28bca20f7e402f0d5d405c134f608d746aac29fc2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4778/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041462227680104448","occurred_at":"2026-04-07T10:24:58Z","failed_at":"e2e","signature_id":"07604109d95a4d7eb7a53c3e5eeb0557b11d04561c2f5e4aa2d12fe00f9ba0d3","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000972a20\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-9f9qss-gzfxn4, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-9f9qss\" in resourcegroup=\"rg-np-version-upgrade-9f9qss-gzfxn4\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-9f9qss-gzfxn4, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-9f9qss\\\" in resourcegroup=\\\"rg-np-version-upgrade-9f9qss-gzfxn4\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000859ef0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-9f9qss-gzfxn4\",\n },\n \u003c*fmt.wrapError | 0xc000df4440\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-9f9qss\\\" in resourcegroup=\\\"rg-np-version-upgrade-9f9qss-gzfxn4\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000972a20\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-9f9qss-gzfxn4, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-9f9qss\" in resourcegroup=\"rg-np-version-upgrade-9f9qss-gzfxn4\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-9f9qss-gzfxn4, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-9f9qss\\\" in resourcegroup=\\\"rg-np-version-upgrade-9f9qss-gzfxn4\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000859ef0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-9f9qss-gzfxn4\",\n },\n \u003c*fmt.wrapError | 0xc000df4440\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-9f9qss\\\" in resourcegroup=\\\"rg-np-version-upgrade-9f9qss-gzfxn4\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041458176968101888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4775,"pr_state":"closed","pr_sha":"3eda0629846526464fbcb6bae8350a6247ae006b","final_merged_sha":"8cab26644417156b4fba886229fd5dd09cf35db9","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T10:08:52Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f2a56de26b20ee5d1f5e2207b49d072f4e0de2ab433726cc1253216031cc5bec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041458176968101888","occurred_at":"2026-04-07T10:08:52Z","failed_at":"e2e","signature_id":"bb8226c56a7c6d26b05d93af133c09c42418e7c13eee3845ced809b8b2d145f4","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00153ab40\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-rd4cq6-8vqk7s, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-rd4cq6\" in resourcegroup=\"rg-np-version-upgrade-rd4cq6-8vqk7s\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-rd4cq6-8vqk7s, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-rd4cq6\\\" in resourcegroup=\\\"rg-np-version-upgrade-rd4cq6-8vqk7s\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0006ca7a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-rd4cq6-8vqk7s\",\n },\n \u003c*fmt.wrapError | 0xc0007786c0\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-rd4cq6\\\" in resourcegroup=\\\"rg-np-version-upgrade-rd4cq6-8vqk7s\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00153ab40\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-rd4cq6-8vqk7s, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-rd4cq6\" in resourcegroup=\"rg-np-version-upgrade-rd4cq6-8vqk7s\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-rd4cq6-8vqk7s, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-rd4cq6\\\" in resourcegroup=\\\"rg-np-version-upgrade-rd4cq6-8vqk7s\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0006ca7a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-rd4cq6-8vqk7s\",\n },\n \u003c*fmt.wrapError | 0xc0007786c0\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-rd4cq6\\\" in resourcegroup=\\\"rg-np-version-upgrade-rd4cq6-8vqk7s\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4719/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041447318581940224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4719,"pr_state":"closed","pr_sha":"3b74828509a1d46ecc3021e200cccedf3b285967","final_merged_sha":"bb8881480b5ead602ebb8a797a1490613a22af93","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T09:25:43Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"981208b57fbb53dadfd1992411280338797625080143095a0ecca3d01d440b30","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4719/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041447318581940224","occurred_at":"2026-04-07T09:25:43Z","failed_at":"e2e","signature_id":"33da2fd5a7d458ba8a822227909e511b4080baef3ee2eeadbc7ff3c67f984dff","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00097f7a0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-5smhlw-pqcgw4, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-5smhlw\" in resourcegroup=\"rg-np-version-upgrade-5smhlw-pqcgw4\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-5smhlw-pqcgw4, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-5smhlw\\\" in resourcegroup=\\\"rg-np-version-upgrade-5smhlw-pqcgw4\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00127d600\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-5smhlw-pqcgw4\",\n },\n \u003c*fmt.wrapError | 0xc0008eb100\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-5smhlw\\\" in resourcegroup=\\\"rg-np-version-upgrade-5smhlw-pqcgw4\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00097f7a0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-5smhlw-pqcgw4, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-5smhlw\" in resourcegroup=\"rg-np-version-upgrade-5smhlw-pqcgw4\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-5smhlw-pqcgw4, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-5smhlw\\\" in resourcegroup=\\\"rg-np-version-upgrade-5smhlw-pqcgw4\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00127d600\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-5smhlw-pqcgw4\",\n },\n \u003c*fmt.wrapError | 0xc0008eb100\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-5smhlw\\\" in resourcegroup=\\\"rg-np-version-upgrade-5smhlw-pqcgw4\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041444636139982848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4599,"pr_state":"closed","pr_sha":"3e1cf837453cde25f5f37e96e2226ca3069bf232","final_merged_sha":"f6ed2d92c3031e65888f81d9423e89db816cc046","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T09:15:04Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d217db651179e57d10e9fb492d7b408aec7bd5ab2db183976e3ff3fbff653259","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4599/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041444636139982848","occurred_at":"2026-04-07T09:15:04Z","failed_at":"e2e","signature_id":"4714acec4ae3710c0f271c1fa4314a34f1cfe5ef7c1fcfddf3d8eace423bd86a","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0008bf1a0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-8pddwt-5cj9qt, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-8pddwt\" in resourcegroup=\"rg-np-version-upgrade-8pddwt-5cj9qt\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-8pddwt-5cj9qt, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-8pddwt\\\" in resourcegroup=\\\"rg-np-version-upgrade-8pddwt-5cj9qt\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000615d50\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-8pddwt-5cj9qt\",\n },\n \u003c*fmt.wrapError | 0xc000d688a0\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-8pddwt\\\" in resourcegroup=\\\"rg-np-version-upgrade-8pddwt-5cj9qt\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0008bf1a0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-8pddwt-5cj9qt, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-8pddwt\" in resourcegroup=\"rg-np-version-upgrade-8pddwt-5cj9qt\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-8pddwt-5cj9qt, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-8pddwt\\\" in resourcegroup=\\\"rg-np-version-upgrade-8pddwt-5cj9qt\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000615d50\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-8pddwt-5cj9qt\",\n },\n \u003c*fmt.wrapError | 0xc000d688a0\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-8pddwt\\\" in resourcegroup=\\\"rg-np-version-upgrade-8pddwt-5cj9qt\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4774/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041429284081373184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4774,"pr_state":"closed","pr_sha":"66f013ad1e0c2e9e611d5404f94edd331ea2e748","final_merged_sha":"1446a17044db64e11af6b6dcc04c4d6c4d89cea7","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T08:14:03Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"36832d630a3b10bb962ab076cc8c10be63f0e0a1a0551c8721f9d65bab336b15","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4774/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041429284081373184","occurred_at":"2026-04-07T08:14:03Z","failed_at":"e2e","signature_id":"8cc6f914f634d5e3fc971358d23f228704c13265e9af97a73c324d6b8b794170","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010d0f30\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-vkjksb-pwpk4j, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-vkjksb\" in resourcegroup=\"rg-np-version-upgrade-vkjksb-pwpk4j\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-vkjksb-pwpk4j, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-vkjksb\\\" in resourcegroup=\\\"rg-np-version-upgrade-vkjksb-pwpk4j\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013ee920\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-vkjksb-pwpk4j\",\n },\n \u003c*fmt.wrapError | 0xc0018f06e0\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-vkjksb\\\" in resourcegroup=\\\"rg-np-version-upgrade-vkjksb-pwpk4j\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010d0f30\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-vkjksb-pwpk4j, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-vkjksb\" in resourcegroup=\"rg-np-version-upgrade-vkjksb-pwpk4j\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-vkjksb-pwpk4j, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-vkjksb\\\" in resourcegroup=\\\"rg-np-version-upgrade-vkjksb-pwpk4j\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013ee920\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-vkjksb-pwpk4j\",\n },\n \u003c*fmt.wrapError | 0xc0018f06e0\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-vkjksb\\\" in resourcegroup=\\\"rg-np-version-upgrade-vkjksb-pwpk4j\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}},{"row_id":"6e1c29b0e8ffd8ef7982bafbc978968f476aef62a3d8d8a4cbbaa51837dc1a12","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4774/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041429284081373184","occurred_at":"2026-04-07T08:14:03Z","failed_at":"e2e","signature_id":"5bb30e1cc3a8f81a01145b833876d1263e8178fefb953e664c88d76dff870928","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0006c71a0\u003e: \n failed to create NodePool np-5j9zvb, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-5j9zvb, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\\\"np-5j9zvb\\\" for cluster \\\"cluster-stable-4-21-5j9zvb\\\" in resourcegroup=\\\"rg-stable-4-21-zqrjr4\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000595a70\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4\",\n },\n \u003c*fmt.wrapError | 0xc0007d4a40\u003e{\n msg: \"failed waiting for nodepool=\\\"np-5j9zvb\\\" for cluster \\\"cluster-stable-4-21-5j9zvb\\\" in resourcegroup=\\\"rg-stable-4-21-zqrjr4\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0006c71a0\u003e: \n failed to create NodePool np-5j9zvb, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-5j9zvb, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\\\"np-5j9zvb\\\" for cluster \\\"cluster-stable-4-21-5j9zvb\\\" in resourcegroup=\\\"rg-stable-4-21-zqrjr4\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000595a70\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4\",\n },\n \u003c*fmt.wrapError | 0xc0007d4a40\u003e{\n msg: \"failed waiting for nodepool=\\\"np-5j9zvb\\\" for cluster \\\"cluster-stable-4-21-5j9zvb\\\" in resourcegroup=\\\"rg-stable-4-21-zqrjr4\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4773/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041427333088284672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4773,"pr_state":"closed","pr_sha":"7b059e7154fb1385ea62a19ab9225e9732f361f6","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T08:06:18Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d8cc78f2f0a76937014bd5266fc0415ddb53e7acc482dd165d0817931523f342","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4773/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041427333088284672","occurred_at":"2026-04-07T08:06:18Z","failed_at":"e2e","signature_id":"dc3ea367baac2ee826790b4dbf107304cbff17eef323ec4cfcdaaa654ecd9995","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00097f3b0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-bkfr5h-qwt4gg, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-bkfr5h\" in resourcegroup=\"rg-np-version-upgrade-bkfr5h-qwt4gg\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-bkfr5h-qwt4gg, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-bkfr5h\\\" in resourcegroup=\\\"rg-np-version-upgrade-bkfr5h-qwt4gg\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000812410\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-bkfr5h-qwt4gg\",\n },\n \u003c*fmt.wrapError | 0xc000823980\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-bkfr5h\\\" in resourcegroup=\\\"rg-np-version-upgrade-bkfr5h-qwt4gg\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00097f3b0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-bkfr5h-qwt4gg, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-bkfr5h\" in resourcegroup=\"rg-np-version-upgrade-bkfr5h-qwt4gg\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-bkfr5h-qwt4gg, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-bkfr5h\\\" in resourcegroup=\\\"rg-np-version-upgrade-bkfr5h-qwt4gg\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000812410\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-bkfr5h-qwt4gg\",\n },\n \u003c*fmt.wrapError | 0xc000823980\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-bkfr5h\\\" in resourcegroup=\\\"rg-np-version-upgrade-bkfr5h-qwt4gg\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041420866239074304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4630,"pr_state":"closed","pr_sha":"319f018742940400aa73f4c69ee0f625c9be7437","final_merged_sha":"7e07b7c9223871a196c30b61870b1a0f20ef1025","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T07:40:36Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"bbd18504b993c8112c045983a796ed5947c2c05c3542008a0b2578d452f579cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041420866239074304","occurred_at":"2026-04-07T07:40:36Z","failed_at":"e2e","signature_id":"79f55b3ef226cbb43a1c597dace13414d527e85c693b20f657fdf6d298c5f046","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0014b5080\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-lg8hsh-vrtrz6, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-lg8hsh\" in resourcegroup=\"rg-np-version-upgrade-lg8hsh-vrtrz6\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-lg8hsh-vrtrz6, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-lg8hsh\\\" in resourcegroup=\\\"rg-np-version-upgrade-lg8hsh-vrtrz6\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000b82530\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-lg8hsh-vrtrz6\",\n },\n \u003c*fmt.wrapError | 0xc000b9c9e0\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-lg8hsh\\\" in resourcegroup=\\\"rg-np-version-upgrade-lg8hsh-vrtrz6\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0014b5080\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-lg8hsh-vrtrz6, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-lg8hsh\" in resourcegroup=\"rg-np-version-upgrade-lg8hsh-vrtrz6\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-lg8hsh-vrtrz6, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-lg8hsh\\\" in resourcegroup=\\\"rg-np-version-upgrade-lg8hsh-vrtrz6\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000b82530\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-lg8hsh-vrtrz6\",\n },\n \u003c*fmt.wrapError | 0xc000b9c9e0\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-lg8hsh\\\" in resourcegroup=\\\"rg-np-version-upgrade-lg8hsh-vrtrz6\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4763/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041420608222269440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4763,"pr_state":"closed","pr_sha":"6396693a06f31224932a8d2a44ca79b04337b133","final_merged_sha":"5cb8e0820f193d7c23a369dbbb0a969d37e7ada3","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T07:39:34Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"327d74a2bbb717aec6bee2c6a9aec8eae3216d2f8de5ff0ded2fa06fdd04898c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4763/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041420608222269440","occurred_at":"2026-04-07T07:39:34Z","failed_at":"e2e","signature_id":"d9219baa712e851708d1dd3fef37213a343a65def073bf7cdd05952d6e4d288c","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:155]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001112f30\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-qt5vb7-8vzh2d, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-qt5vb7\" in resourcegroup=\"rg-np-version-upgrade-qt5vb7-8vzh2d\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-qt5vb7-8vzh2d, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-qt5vb7\\\" in resourcegroup=\\\"rg-np-version-upgrade-qt5vb7-8vzh2d\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00033a510\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-qt5vb7-8vzh2d\",\n },\n \u003c*fmt.wrapError | 0xc0012805e0\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-qt5vb7\\\" in resourcegroup=\\\"rg-np-version-upgrade-qt5vb7-8vzh2d\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:155]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001112f30\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-qt5vb7-8vzh2d, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-qt5vb7\" in resourcegroup=\"rg-np-version-upgrade-qt5vb7-8vzh2d\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-qt5vb7-8vzh2d, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-qt5vb7\\\" in resourcegroup=\\\"rg-np-version-upgrade-qt5vb7-8vzh2d\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00033a510\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-qt5vb7-8vzh2d\",\n },\n \u003c*fmt.wrapError | 0xc0012805e0\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-qt5vb7\\\" in resourcegroup=\\\"rg-np-version-upgrade-qt5vb7-8vzh2d\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}},{"row_id":"cbd0f0b3263ae4e96939d09a80dac26309d7ae53b002291cb6f0c9be47f78f5b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4763/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041420608222269440","occurred_at":"2026-04-07T07:39:34Z","failed_at":"e2e","signature_id":"f2191e25b3e1bfd623e5187eac8c488e0c4fab0d49e65850af6997faae0540c7","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:235]: Expected success, but got an error:\n \u003c*errors.errorString | 0xc00069b740\u003e: \n expected 3 nodes, found 4\n {\n s: \"expected 3 nodes, found 4\",\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:235]: Expected success, but got an error:\n \u003c*errors.errorString | 0xc00069b740\u003e: \n expected 3 nodes, found 4\n {\n s: \"expected 3 nodes, found 4\",\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0d7155fc1d72ecaf6add588cb628cbde23fa016e9c93c3f65453d6fef586f5c","failure_pattern":"expected 3 nodes, found 4","search_query":"expected 3 nodes, found 4"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","b0d7155fc1d72ecaf6add588cb628cbde23fa016e9c93c3f65453d6fef586f5c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4772/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041417899385556992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4772,"pr_state":"closed","pr_sha":"14c315a53cc8ef7296ac80ba8f1749759c564d48","final_merged_sha":"14c315a53cc8ef7296ac80ba8f1749759c564d48","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T07:28:49Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"27b3b14ab759f5e1f51b7f4f80752b503149d895df6f404443b60040a278db98","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4772/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041417899385556992","occurred_at":"2026-04-07T07:28:49Z","failed_at":"e2e","signature_id":"b23482115a17e8d00ff45dc3f9a9e11ec47effe3af233b44cf6bcbff90265be5","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001550450\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-j8c5t7-swgw2c, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-j8c5t7\" in resourcegroup=\"rg-np-version-upgrade-j8c5t7-swgw2c\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-j8c5t7-swgw2c, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-j8c5t7\\\" in resourcegroup=\\\"rg-np-version-upgrade-j8c5t7-swgw2c\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00154bd00\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-j8c5t7-swgw2c\",\n },\n \u003c*fmt.wrapError | 0xc000a3e260\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-j8c5t7\\\" in resourcegroup=\\\"rg-np-version-upgrade-j8c5t7-swgw2c\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001550450\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-j8c5t7-swgw2c, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-j8c5t7\" in resourcegroup=\"rg-np-version-upgrade-j8c5t7-swgw2c\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-j8c5t7-swgw2c, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-j8c5t7\\\" in resourcegroup=\\\"rg-np-version-upgrade-j8c5t7-swgw2c\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00154bd00\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-j8c5t7-swgw2c\",\n },\n \u003c*fmt.wrapError | 0xc000a3e260\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-j8c5t7\\\" in resourcegroup=\\\"rg-np-version-upgrade-j8c5t7-swgw2c\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4700/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041415507445288960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4700,"pr_state":"closed","pr_sha":"0409e4102b795d6ec59de1062b245d0bb107117d","final_merged_sha":"f2c81341ce3bebdf6041e299c18aac51d3a3a807","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T07:19:19Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"60e43fdc548300c98890601b156678275af663b6990075fc7726138bb90337cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4700/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041415507445288960","occurred_at":"2026-04-07T07:19:19Z","failed_at":"e2e","signature_id":"0cee71175c67584767c62e32700d8b478b5abb5193110a5aeb98f597f1b44b71","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012ae240\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-fgp88h\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-fgp88h, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-fgp88h\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-fgp88h, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00098cb50\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-fgp88h\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:183]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012ae240\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-fgp88h\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-fgp88h, error: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"np-1\\\" in cluster=\\\"cluster-np-labels-taints\\\" resourcegroup=\\\"rg-np-labels-taints-fgp88h\\\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-fgp88h, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00098cb50\u003e{\n s: \"timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-fgp88h\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-82qzvg\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-82qzvg, error: context deadline exceeded"}},{"row_id":"82b7928adb41178e4ab47c7cd74d8b6e385da7cfd7f6d40df2069c258c06e7c6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4700/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041415507445288960","occurred_at":"2026-04-07T07:19:19Z","failed_at":"e2e","signature_id":"74a9c87c5620408ec6024466a2cbfc7862e7454efaa6c6084540b66ec2ef2b40","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:281]: Timed out after 30.001s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc001352320\u003e: \n permission checks failed: expected allowed but denied: list flowcontrol.apiserver.k8s.io/flowschemas (reason: )\n expected allowed but denied: list flowcontrol.apiserver.k8s.io/prioritylevelconfigurations (reason: )\n expected allowed but denied: list monitoring.coreos.com/prometheusrules (reason: )\n expected allowed but denied: list monitoring.coreos.com/servicemonitors (reason: )\n expected allowed but denied: list machine.openshift.io/machines (reason: )\n expected allowed but denied: list machine.openshift.io/machinesets (reason: )\n expected allowed but denied: list operator.openshift.io/ingresscontrollers (reason: )\n {\n msg: \"permission checks failed: expected allowed but denied: list flowcontrol.apiserver.k8s.io/flowschemas (reason: )\\nexpected allowed but denied: list flowcontrol.apiserver.k8s.io/prioritylevelconfigurations (reason: )\\nexpected allowed but denied: list monitoring.coreos.com/prometheusrules (reason: )\\nexpected allowed but denied: list monitoring.coreos.com/servicemonitors (reason: )\\nexpected allowed but denied: list machine.openshift.io/machines (reason: )\\nexpected allowed but denied: list machine.openshift.io/machinesets (reason: )\\nexpected allowed but denied: list operator.openshift.io/ingresscontrollers (reason: )\",\n err: \u003c*errors.joinError | 0xc000f25f38\u003e{\n errs: [\n \u003c*errors.errorString | 0xc000f81780\u003e{\n s: \"expected allowed but denied: list flowcontrol.apiserver.k8s.io/flowschemas (reason: )\",\n },\n \u003c*errors.errorString | 0xc000b17370\u003e{\n s: \"expected allowed but denied: list flowcontrol.apiserver.k8s.io/prioritylevelconfigurations (reason: )\",\n },\n \u003c*errors.errorString | 0xc000c3c300\u003e{\n s: \"expected allowed but denied: list monitoring.coreos.com/prometheusrules (reason: )\",\n },\n \u003c*errors.errorString | 0xc000f81b90\u003e{\n s: \"expected allowed but denied: list monitoring.coreos.com/servicemonitors (reason: )\",\n },\n \u003c*errors.errorString | 0xc000b17760\u003e{\n s: \"expected allowed but denied: list machine.openshift.io/machines (reason: )\",\n },\n \u003c*errors.errorString | 0xc000b17910\u003e{\n s: \"expected allowed but denied: list machine.openshift.io/machinesets (reason: )\",\n },\n \u003c*errors.errorString | 0xc000c3c4d0\u003e{\n s: \"expected allowed but denied: list operator.openshift.io/ingresscontrollers (reason: )\",\n },\n ],\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:281]: Timed out after 30.001s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc001352320\u003e: \n permission checks failed: expected allowed but denied: list flowcontrol.apiserver.k8s.io/flowschemas (reason: )\n expected allowed but denied: list flowcontrol.apiserver.k8s.io/prioritylevelconfigurations (reason: )\n expected allowed but denied: list monitoring.coreos.com/prometheusrules (reason: )\n expected allowed but denied: list monitoring.coreos.com/servicemonitors (reason: )\n expected allowed but denied: list machine.openshift.io/machines (reason: )\n expected allowed but denied: list machine.openshift.io/machinesets (reason: )\n expected allowed but denied: list operator.openshift.io/ingresscontrollers (reason: )\n {\n msg: \"permission checks failed: expected allowed but denied: list flowcontrol.apiserver.k8s.io/flowschemas (reason: )\\nexpected allowed but denied: list flowcontrol.apiserver.k8s.io/prioritylevelconfigurations (reason: )\\nexpected allowed but denied: list monitoring.coreos.com/prometheusrules (reason: )\\nexpected allowed but denied: list monitoring.coreos.com/servicemonitors (reason: )\\nexpected allowed but denied: list machine.openshift.io/machines (reason: )\\nexpected allowed but denied: list machine.openshift.io/machinesets (reason: )\\nexpected allowed but denied: list operator.openshift.io/ingresscontrollers (reason: )\",\n err: \u003c*errors.joinError | 0xc000f25f38\u003e{\n errs: [\n \u003c*errors.errorString | 0xc000f81780\u003e{\n s: \"expected allowed but denied: list flowcontrol.apiserver.k8s.io/flowschemas (reason: )\",\n },\n \u003c*errors.errorString | 0xc000b17370\u003e{\n s: \"expected allowed but denied: list flowcontrol.apiserver.k8s.io/prioritylevelconfigurations (reason: )\",\n },\n \u003c*errors.errorString | 0xc000c3c300\u003e{\n s: \"expected allowed but denied: list monitoring.coreos.com/prometheusrules (reason: )\",\n },\n \u003c*errors.errorString | 0xc000f81b90\u003e{\n s: \"expected allowed but denied: list monitoring.coreos.com/servicemonitors (reason: )\",\n },\n \u003c*errors.errorString | 0xc000b17760\u003e{\n s: \"expected allowed but denied: list machine.openshift.io/machines (reason: )\",\n },\n \u003c*errors.errorString | 0xc000b17910\u003e{\n s: \"expected allowed but denied: list machine.openshift.io/machinesets (reason: )\",\n },\n \u003c*errors.errorString | 0xc000c3c4d0\u003e{\n s: \"expected allowed but denied: list operator.openshift.io/ingresscontrollers (reason: )\",\n },\n ],\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cada5ab0a8f8fe508d400c0c0ce723760422f5303a8840acbdf56fa572eae5f2","failure_pattern":"permission checks failed: expected allowed but denied: list flowcontrol","search_query":"permission checks failed: expected allowed but denied: list flowcontrol"}},{"row_id":"e4de0f1392e39c9ac5b7e7b25de3244c289ce5a4bdb606029e0165648fa7cf03","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4700/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041415507445288960","occurred_at":"2026-04-07T07:19:19Z","failed_at":"e2e","signature_id":"a9a7e0dab84c771e91bfc8beca701e4916cf7e75e66dac49d9cef25149991ffc","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001486c60\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-s4jf4r-qp5ktw, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-s4jf4r\" in resourcegroup=\"rg-np-version-upgrade-s4jf4r-qp5ktw\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-s4jf4r-qp5ktw, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-s4jf4r\\\" in resourcegroup=\\\"rg-np-version-upgrade-s4jf4r-qp5ktw\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013f5b40\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-s4jf4r-qp5ktw\",\n },\n \u003c*fmt.wrapError | 0xc00097c540\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-s4jf4r\\\" in resourcegroup=\\\"rg-np-version-upgrade-s4jf4r-qp5ktw\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001486c60\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-s4jf4r-qp5ktw, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-s4jf4r\" in resourcegroup=\"rg-np-version-upgrade-s4jf4r-qp5ktw\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-s4jf4r-qp5ktw, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-s4jf4r\\\" in resourcegroup=\\\"rg-np-version-upgrade-s4jf4r-qp5ktw\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013f5b40\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-s4jf4r-qp5ktw\",\n },\n \u003c*fmt.wrapError | 0xc00097c540\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-s4jf4r\\\" in resourcegroup=\\\"rg-np-version-upgrade-s4jf4r-qp5ktw\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","cada5ab0a8f8fe508d400c0c0ce723760422f5303a8840acbdf56fa572eae5f2"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4771/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041410465094963200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4771,"pr_state":"closed","pr_sha":"5b35760875077d9d675849752d78ac6e35863050","final_merged_sha":"5b35760875077d9d675849752d78ac6e35863050","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T06:59:17Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"de0544140d32b671b94d5fd26e0de97c3d17093f620cefe7af85f7110c0b6415","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4771/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041410465094963200","occurred_at":"2026-04-07T06:59:17Z","failed_at":"e2e","signature_id":"53288fd6bec0d4e3daaaa13398d6910d74b967e292962287c3affbab32ad328e","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000974900\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-rqvh4j-mhhhl4, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-rqvh4j\" in resourcegroup=\"rg-np-version-upgrade-rqvh4j-mhhhl4\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-rqvh4j-mhhhl4, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-rqvh4j\\\" in resourcegroup=\\\"rg-np-version-upgrade-rqvh4j-mhhhl4\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f04430\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-rqvh4j-mhhhl4\",\n },\n \u003c*fmt.wrapError | 0xc001308300\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-rqvh4j\\\" in resourcegroup=\\\"rg-np-version-upgrade-rqvh4j-mhhhl4\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000974900\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-rqvh4j-mhhhl4, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-rqvh4j\" in resourcegroup=\"rg-np-version-upgrade-rqvh4j-mhhhl4\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-rqvh4j-mhhhl4, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-rqvh4j\\\" in resourcegroup=\\\"rg-np-version-upgrade-rqvh4j-mhhhl4\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f04430\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-rqvh4j-mhhhl4\",\n },\n \u003c*fmt.wrapError | 0xc001308300\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-rqvh4j\\\" in resourcegroup=\\\"rg-np-version-upgrade-rqvh4j-mhhhl4\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041407133697708032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4640,"pr_state":"closed","pr_sha":"049af68bf5c7be01cd9616938b5c5adcc3c79ba7","final_merged_sha":"e23d7c6b5233dccc23b0d4b24f1d727a143ec398","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T06:46:02Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"58d1f36e2f42565ca777a76d5989ad536d123a85a4e09126e3808d20f750e38b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041407133697708032","occurred_at":"2026-04-07T06:46:02Z","failed_at":"provision","signature_id":"193a7f4359d91d145d9186a00482b7e87740b4491ecfc11871a2a8ddef209d2f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/service/deploy","test_suite":"step graph","failure_text":"time=2026-04-07T07:32:01.123Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-04-07T07:32:01.627Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:050da8f5-46c5-4d7d-a866-f2e1d694ae69 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j7708032] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j7708032.hcpsvc.osadev.cloud] image:map[digest:sha256:8d70336c30fca10dbcc85844ab84131c0a320d6935ec102f5852f3aca80477c2 registry:arohcpsvcdev.azurecr.io repository:ci-op-c9qsmib2/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-c9qsmib2/pipeline:pull workloadIdentityClientId:33398a2d-ec26-4f8f-b2ee-2d8f33509710 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:67717d2a-5a54-4003-93d4-5172d763f82b workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-04-07T07:32:01.627Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-04-07T07:32:01.990Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy namespace=sessiongate\ntime=2026-04-07T07:32:02.278Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy dryRun=false\ntime=2026-04-07T07:32:02.278Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-04-07T07:32:02.396Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-04-07T07:32:11.442Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.43:15017, code 500: 500 Internal Server Error\\nInternal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.51:15017, code 500: 500 Internal Server Error\"\ntime=2026-04-07T07:32:11.442Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-04-07T07:32:11.442Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-04-07T07:32:13.500Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy release=sessiongate namespace=sessiongate status=failed description=\"Release \\\"sessiongate\\\" failed: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.43:15017, code 500: 500 Internal Server Error\\nInternal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.51:15017, code 500: 500 Internal Server Error\" values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:050da8f5-46c5-4d7d-a866-f2e1d694ae69 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j7708032] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j7708032.hcpsvc.osadev.cloud] image:map[digest:sha256:8d70336c30fca10dbcc85844ab84131c0a320d6935ec102f5852f3aca80477c2 registry:arohcpsvcdev.azurecr.io repository:ci-op-c9qsmib2/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-c9qsmib2/pipeline:pull workloadIdentityClientId:33398a2d-ec26-4f8f-b2ee-2d8f33509710 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:67717d2a-5a54-4003-93d4-5172d763f82b workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-04-07T07:32:13.502Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/6RUQW/aTBC9+1esfAoSaxG+T41ElQNNKlWqVCHCKVUP6/XDbFjvWDNrEBE/vnKAxiIEUvW2fn775r3xeBKPqBhCDVuIulWFiSaa3OOK8ifY+N2FYiSRXSj7agf9MBX+QFghvCBSG3uAez8TpZRKH8ArZzG2lpoQ075KBSKOQmki3jzu7tz5RiJ4Sv6YofdnXZlgSlTYSR4ufpvNJlNqTgifrPMAy4gTppUrwHfeiLypZ+sWMkvRTqIj7ULJaHl7q41Eqqb77t1j7oKLjkJHR7KOYGaY9MLWmXluGJmlquv/VGJLITJ5D34nxflL2sMUYA0Pe2TsWOSLC4UL5Rmt/CTj7zSO/FzQ3I/Px77nPWpPm8NQ/ANfW3B0c2fbM2POkMWu/+8OwtjypPG+k79uvL+QbtzEBbF7Nm0nJuSd3bwU8Z7WukJkZ0XL5SBndQLFxQkLya/PybLJwQER8rX9hyXZqvUCDBVdBYmmqlWOuAaCuipMRAtfpcPB8JMe/K8HN7PBzei/4WgwfEx7KsvUGc719WPa6yVb9UQuqKULxa0LAdzZPBTU68LpbprjFZNsVc3Uvn012lc1FTvyx0T6ylPZVxVETNlKEhdglW864QuI/R0AAP//Uu+zzx8FAAA=\"\ntime=2026-04-07T07:32:13.505Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-04-07T07:32:13.505Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.43:15017, code 500: 500 Internal Server Error\\nInternal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.51:15017, code 500: 500 Internal Server Error\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a83e14f56d71f4d9dfea9f79e4a2b2d28df34e80a7e1b1939e2b98b6de4b1e22","failure_pattern":"Internal error occurred: failed calling webhook \"validation.istio.io\": failed to call webhook: Post \"\u003curl\u003e\": proxy error from localhost:9443 while dialing \u003cip\u003e:\u003cport\u003e, code 500: 500 Internal Server Error","search_query":"Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.51:15017, code 500: 500 Internal Server Error"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a83e14f56d71f4d9dfea9f79e4a2b2d28df34e80a7e1b1939e2b98b6de4b1e22"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041406425963433984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4770,"pr_state":"closed","pr_sha":"0469cc10771411b52260544ac6b5a788cad9d043","final_merged_sha":"0469cc10771411b52260544ac6b5a788cad9d043","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T06:43:14Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"ab0a0315eb55e10ac8239fe1451234d6ce49cc2531ea06e6e1750157163c0716","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041406425963433984","occurred_at":"2026-04-07T06:43:14Z","failed_at":"e2e","signature_id":"9f4a8907dca16b9ecddbb321b4c75ff09203188649a062a0577d1b48b6454a6f","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011b3410\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-lhgchq-vjdd4r, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-lhgchq\" in resourcegroup=\"rg-np-version-upgrade-lhgchq-vjdd4r\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-lhgchq-vjdd4r, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-lhgchq\\\" in resourcegroup=\\\"rg-np-version-upgrade-lhgchq-vjdd4r\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000922320\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-lhgchq-vjdd4r\",\n },\n \u003c*fmt.wrapError | 0xc000e687c0\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-lhgchq\\\" in resourcegroup=\\\"rg-np-version-upgrade-lhgchq-vjdd4r\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011b3410\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-lhgchq-vjdd4r, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-lhgchq\" in resourcegroup=\"rg-np-version-upgrade-lhgchq-vjdd4r\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-lhgchq-vjdd4r, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-lhgchq\\\" in resourcegroup=\\\"rg-np-version-upgrade-lhgchq-vjdd4r\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000922320\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-lhgchq-vjdd4r\",\n },\n \u003c*fmt.wrapError | 0xc000e687c0\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-lhgchq\\\" in resourcegroup=\\\"rg-np-version-upgrade-lhgchq-vjdd4r\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041406261966147584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4582,"pr_state":"closed","pr_sha":"453429fc550d57d7915964340a777a91b19a1208","final_merged_sha":"453429fc550d57d7915964340a777a91b19a1208","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T06:42:34Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"10c9d007fb78ebd25cd1bbf849e454f3798511ce02d5da39e1e8da07612f4f20","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4582/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041406261966147584","occurred_at":"2026-04-07T06:42:34Z","failed_at":"e2e","signature_id":"38e8127990e1b255ab87796250d20916b99e3259665a732499f795f16367a870","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009156b0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-26p857-vxdhdm, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-26p857\" in resourcegroup=\"rg-np-version-upgrade-26p857-vxdhdm\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-26p857-vxdhdm, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-26p857\\\" in resourcegroup=\\\"rg-np-version-upgrade-26p857-vxdhdm\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00112c8d0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-26p857-vxdhdm\",\n },\n \u003c*fmt.wrapError | 0xc00096f240\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-26p857\\\" in resourcegroup=\\\"rg-np-version-upgrade-26p857-vxdhdm\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009156b0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-26p857-vxdhdm, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-26p857\" in resourcegroup=\"rg-np-version-upgrade-26p857-vxdhdm\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-26p857-vxdhdm, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-26p857\\\" in resourcegroup=\\\"rg-np-version-upgrade-26p857-vxdhdm\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00112c8d0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-26p857-vxdhdm\",\n },\n \u003c*fmt.wrapError | 0xc00096f240\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-26p857\\\" in resourcegroup=\\\"rg-np-version-upgrade-26p857-vxdhdm\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4549/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041392080730722304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4549,"pr_state":"closed","pr_sha":"8ac51d708827eb5c10759a0f8669b57677375e65","final_merged_sha":"8ac51d708827eb5c10759a0f8669b57677375e65","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T05:46:13Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"5ab97a1c6a4d0cad278882c23734a465fef0c2a7cb63ca89869fac5d8a08c6ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4549/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041392080730722304","occurred_at":"2026-04-07T05:46:13Z","failed_at":"e2e","signature_id":"ba4afb961a4ecb0bcd05aeddd2fd094ee8d787ba2bb0751ae4fd3d4b9428db6b","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001614f90\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-g7vd82-n5vhfx, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-g7vd82\" in resourcegroup=\"rg-np-version-upgrade-g7vd82-n5vhfx\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-g7vd82-n5vhfx, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-g7vd82\\\" in resourcegroup=\\\"rg-np-version-upgrade-g7vd82-n5vhfx\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f05ed0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-g7vd82-n5vhfx\",\n },\n \u003c*fmt.wrapError | 0xc000852b80\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-g7vd82\\\" in resourcegroup=\\\"rg-np-version-upgrade-g7vd82-n5vhfx\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001614f90\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-g7vd82-n5vhfx, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-g7vd82\" in resourcegroup=\"rg-np-version-upgrade-g7vd82-n5vhfx\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-g7vd82-n5vhfx, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-g7vd82\\\" in resourcegroup=\\\"rg-np-version-upgrade-g7vd82-n5vhfx\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f05ed0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-g7vd82-n5vhfx\",\n },\n \u003c*fmt.wrapError | 0xc000852b80\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-g7vd82\\\" in resourcegroup=\\\"rg-np-version-upgrade-g7vd82-n5vhfx\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041384066896367616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4630,"pr_state":"closed","pr_sha":"319f018742940400aa73f4c69ee0f625c9be7437","final_merged_sha":"7e07b7c9223871a196c30b61870b1a0f20ef1025","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T05:14:08Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"6060ad0d3c61e858d99bff80d33fb709b65bbdc687cc242ca40d28c3dd8b5587","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041384066896367616","occurred_at":"2026-04-07T05:14:08Z","failed_at":"e2e","signature_id":"b5d16c8b12ac066bb58ebf1698f548dfd72c9383f30e390628c368803d7eeb62","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0006cccf0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-ddh85g-996lfr, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-ddh85g\" in resourcegroup=\"rg-np-version-upgrade-ddh85g-996lfr\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-ddh85g-996lfr, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-ddh85g\\\" in resourcegroup=\\\"rg-np-version-upgrade-ddh85g-996lfr\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000c51210\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-ddh85g-996lfr\",\n },\n \u003c*fmt.wrapError | 0xc000d84640\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-ddh85g\\\" in resourcegroup=\\\"rg-np-version-upgrade-ddh85g-996lfr\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0006cccf0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-ddh85g-996lfr, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-ddh85g\" in resourcegroup=\"rg-np-version-upgrade-ddh85g-996lfr\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-ddh85g-996lfr, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-ddh85g\\\" in resourcegroup=\\\"rg-np-version-upgrade-ddh85g-996lfr\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000c51210\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-ddh85g-996lfr\",\n },\n \u003c*fmt.wrapError | 0xc000d84640\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-ddh85g\\\" in resourcegroup=\\\"rg-np-version-upgrade-ddh85g-996lfr\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041362052848553984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4770,"pr_state":"closed","pr_sha":"0469cc10771411b52260544ac6b5a788cad9d043","final_merged_sha":"0469cc10771411b52260544ac6b5a788cad9d043","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T03:46:54Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1daa46e3f89402a81bb5e716adf32a2ecaae1296df35f4730f354d270c74e424","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041362052848553984","occurred_at":"2026-04-07T03:46:54Z","failed_at":"e2e","signature_id":"a491cd3b0276f504d19f80e48a64875d8cc00891a8f4f53aa98b5ae64fca8243","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009637d0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-v8c9qm-glzmd5, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-v8c9qm\" in resourcegroup=\"rg-np-version-upgrade-v8c9qm-glzmd5\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-v8c9qm-glzmd5, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-v8c9qm\\\" in resourcegroup=\\\"rg-np-version-upgrade-v8c9qm-glzmd5\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fdda60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-v8c9qm-glzmd5\",\n },\n \u003c*fmt.wrapError | 0xc000c44d40\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-v8c9qm\\\" in resourcegroup=\\\"rg-np-version-upgrade-v8c9qm-glzmd5\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009637d0\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-v8c9qm-glzmd5, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-v8c9qm\" in resourcegroup=\"rg-np-version-upgrade-v8c9qm-glzmd5\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-v8c9qm-glzmd5, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-v8c9qm\\\" in resourcegroup=\\\"rg-np-version-upgrade-v8c9qm-glzmd5\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fdda60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-v8c9qm-glzmd5\",\n },\n \u003c*fmt.wrapError | 0xc000c44d40\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-v8c9qm\\\" in resourcegroup=\\\"rg-np-version-upgrade-v8c9qm-glzmd5\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}},{"row_id":"f28eb20e0470a0f433981789f3512ad58e9a04c186fd75615bfe25c5d54531bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4770/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041362052848553984","occurred_at":"2026-04-07T03:46:54Z","failed_at":"e2e","signature_id":"c02921d05e7089b501a7d646233698151beb3c572e784eb30e6b98762e669288","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:108]: Unexpected error:\n \u003c*errors.joinError | 0xc00103fb48\u003e: \n VerifyAllAPIServicesAvailable failed: the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000830a60\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then\",\n err: \u003c*errors.errorString | 0xc000ab2280\u003e{\n s: \"the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then\",\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:108]: Unexpected error:\n \u003c*errors.joinError | 0xc00103fb48\u003e: \n VerifyAllAPIServicesAvailable failed: the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000830a60\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then\",\n err: \u003c*errors.errorString | 0xc000ab2280\u003e{\n s: \"the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then\",\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"53c8663bb9391640cf5c05f563d13119c0eb49450a0c9d118656d27d9b9f12e7","failure_pattern":"VerifyAllAPIServicesAvailable failed: the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then","search_query":"VerifyAllAPIServicesAvailable failed: the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["53c8663bb9391640cf5c05f563d13119c0eb49450a0c9d118656d27d9b9f12e7","a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4549/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041344315120160768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4549,"pr_state":"closed","pr_sha":"8ac51d708827eb5c10759a0f8669b57677375e65","final_merged_sha":"8ac51d708827eb5c10759a0f8669b57677375e65","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T02:36:25Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f6010c9ce912aafc137af45d1daed86ff7fe8cb298113914ffa873ce8a50d46d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4549/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041344315120160768","occurred_at":"2026-04-07T02:36:25Z","failed_at":"e2e","signature_id":"fd188e3e01088845af5ed7604d0e36024275083fe23cb51f714d200b7e32aa02","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009faf00\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-cvx4gs-h69ltv, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-cvx4gs\" in resourcegroup=\"rg-np-version-upgrade-cvx4gs-h69ltv\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-cvx4gs-h69ltv, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-cvx4gs\\\" in resourcegroup=\\\"rg-np-version-upgrade-cvx4gs-h69ltv\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000746bb0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-cvx4gs-h69ltv\",\n },\n \u003c*fmt.wrapError | 0xc0008f8d00\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-cvx4gs\\\" in resourcegroup=\\\"rg-np-version-upgrade-cvx4gs-h69ltv\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009faf00\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-cvx4gs-h69ltv, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-cvx4gs\" in resourcegroup=\"rg-np-version-upgrade-cvx4gs-h69ltv\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-cvx4gs-h69ltv, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-cvx4gs\\\" in resourcegroup=\\\"rg-np-version-upgrade-cvx4gs-h69ltv\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000746bb0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-cvx4gs-h69ltv\",\n },\n \u003c*fmt.wrapError | 0xc0008f8d00\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-cvx4gs\\\" in resourcegroup=\\\"rg-np-version-upgrade-cvx4gs-h69ltv\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041319917524881408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4004,"pr_state":"closed","pr_sha":"ff5b88c96712835bead8c845c79f61dfb754f685","final_merged_sha":"9feee09ab6996cfe7dd433b262345799b7851593","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T00:59:28Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"46843b9ece0a086d2b9bfe7de436d0217425800ad7dc7c74af4cf0bc834baf96","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041319917524881408","occurred_at":"2026-04-07T00:59:28Z","failed_at":"e2e","signature_id":"7189c9d77abbfd8a63e92ed78869a8d0f55b3f6cf0b50bdb1164d7a2f117036c","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000757050\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-652n4h-lfjzg2, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-652n4h\" in resourcegroup=\"rg-np-version-upgrade-652n4h-lfjzg2\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-652n4h-lfjzg2, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-652n4h\\\" in resourcegroup=\\\"rg-np-version-upgrade-652n4h-lfjzg2\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0015f1cc0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-652n4h-lfjzg2\",\n },\n \u003c*fmt.wrapError | 0xc000c7a5e0\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-652n4h\\\" in resourcegroup=\\\"rg-np-version-upgrade-652n4h-lfjzg2\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000757050\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-652n4h-lfjzg2, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-652n4h\" in resourcegroup=\"rg-np-version-upgrade-652n4h-lfjzg2\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-652n4h-lfjzg2, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-652n4h\\\" in resourcegroup=\\\"rg-np-version-upgrade-652n4h-lfjzg2\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0015f1cc0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-652n4h-lfjzg2\",\n },\n \u003c*fmt.wrapError | 0xc000c7a5e0\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-652n4h\\\" in resourcegroup=\\\"rg-np-version-upgrade-652n4h-lfjzg2\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4210/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041308047103496192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4210,"pr_state":"closed","pr_sha":"205d2476345b6a8afc6a7a6b9f8ae6a5026dfc74","final_merged_sha":"ef912c114f3891ce814ee96ef6545953f99c7540","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-04-07T00:12:18Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c502a67dd7179dab0e051603977004bc836d0f3542c5584c9a7f79ab2c16e6ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4210/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2041308047103496192","occurred_at":"2026-04-07T00:12:18Z","failed_at":"e2e","signature_id":"842e4721ccc75c1f3b8454624f6bef7da614a0f7e01615b0c577858f03385b68","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001789200\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-dkddpp-7s2skb, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-dkddpp\" in resourcegroup=\"rg-np-version-upgrade-dkddpp-7s2skb\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-dkddpp-7s2skb, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-dkddpp\\\" in resourcegroup=\\\"rg-np-version-upgrade-dkddpp-7s2skb\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f499d0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-dkddpp-7s2skb\",\n },\n \u003c*fmt.wrapError | 0xc000aa4e40\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-dkddpp\\\" in resourcegroup=\\\"rg-np-version-upgrade-dkddpp-7s2skb\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001789200\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-dkddpp-7s2skb, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-dkddpp\" in resourcegroup=\"rg-np-version-upgrade-dkddpp-7s2skb\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-dkddpp-7s2skb, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-dkddpp\\\" in resourcegroup=\\\"rg-np-version-upgrade-dkddpp-7s2skb\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f499d0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-dkddpp-7s2skb\",\n },\n \u003c*fmt.wrapError | 0xc000aa4e40\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-dkddpp\\\" in resourcegroup=\\\"rg-np-version-upgrade-dkddpp-7s2skb\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-5j9zvb in resource group rg-stable-4-21-zqrjr4, error: failed waiting for nodepool=\"np-5j9zvb\" for cluster \"cluster-stable-4-21-5j9zvb\" in resourcegroup=\"rg-stable-4-21-zqrjr4\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}