{"meta":{"date":"2026-03-26","timezone":"UTC","generated_at":"2026-10-31T23:15:27Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":67,"failed_runs":37,"runs_with_occurrences":37,"runs_with_failure_pattern_matches":15,"runs_with_unmatched_occurrences":22,"runs_without_occurrence_rows":30,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4652/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037297014319878144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4652,"pr_state":"closed","pr_sha":"57b0fed262d1ac83f97d560307f728594f1e3f1d","final_merged_sha":"ef0f1b19b5b4a0415c36e782700ec263a51ca311","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-26T22:33:53Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3258a892f5c7a3153ebe60b5f92f2cdf053e6b9b32b012f349f44c139403fcd0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4652/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037297014319878144","occurred_at":"2026-03-26T22:33:53Z","failed_at":"e2e","signature_id":"00f7b6b5022263e6d185c794022d2c6b5f27ecf6f9854a1799e6e112ab5a01a8","test_name":"Customer should perform a control plane y-stream upgrade from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:171]: Timed out after 2700.006s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000114e88\u003e: \n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0, targetMinor=4.21.0) failed: clusterversion status.history has no version in target minor \"4.21.0\"\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a1a7a0\u003e{\n msg: \"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0, targetMinor=4.21.0) failed: clusterversion status.history has no version in target minor \\\"4.21.0\\\"\",\n err: \u003c*errors.errorString | 0xc0007aa580\u003e{\n s: \"clusterversion status.history has no version in target minor \\\"4.21.0\\\"\",\n },\n },\n ],\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:171]: Timed out after 2700.006s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000114e88\u003e: \n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0, targetMinor=4.21.0) failed: clusterversion status.history has no version in target minor \"4.21.0\"\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a1a7a0\u003e{\n msg: \"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.20.0, targetMinor=4.21.0) failed: clusterversion status.history has no version in target minor \\\"4.21.0\\\"\",\n err: \u003c*errors.errorString | 0xc0007aa580\u003e{\n s: \"clusterversion status.history has no version in target minor \\\"4.21.0\\\"\",\n },\n },\n ],\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b952bd87cc064f775e96c64b34d1d595213c1b657abf569b98d2c49605644533","failure_pattern":"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4","search_query":"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b952bd87cc064f775e96c64b34d1d595213c1b657abf569b98d2c49605644533"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037296632252338176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4640,"pr_state":"closed","pr_sha":"d3d1e552feb01908dd498b431aabb88c99d49367","final_merged_sha":"e23d7c6b5233dccc23b0d4b24f1d727a143ec398","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-26T22:32:22Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"626558ae428619c55641dc142a33213ca3b0d11898b2c4450aa97f1ce6ee6da6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037296632252338176","occurred_at":"2026-03-26T22:32:22Z","failed_at":"e2e","signature_id":"ece868d3ee27fb756564e3c1e3b5e3434dffdd956d652acc50a3f0a0e6c93454","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:281]: Timed out after 30.001s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc001030540\u003e: \n read access verification failed: get pods -n kube-system: pods is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"pods\" in API group \"\" in the namespace \"kube-system\"\n get configmaps -n kube-system: configmaps is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"configmaps\" in API group \"\" in the namespace \"kube-system\"\n {\n msg: \"read access verification failed: get pods -n kube-system: pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\\nget configmaps -n kube-system: configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.joinError | 0xc000fb1de8\u003e{\n errs: [\n \u003c*fmt.wrapError | 0xc001100b60\u003e{\n msg: \"get pods -n kube-system: pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.StatusError | 0xc001106b40\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n Reason: \"Forbidden\",\n Details: {Name: \"\", Group: \"\", Kind: \"pods\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n },\n },\n \u003c*fmt.wrapError | 0xc0010304e0\u003e{\n msg: \"get configmaps -n kube-system: configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.StatusError | 0xc001020780\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n Reason: \"Forbidden\",\n Details: {Name: \"\", Group: \"\", Kind: \"configmaps\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n },\n },\n ],\n },\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:281]: Timed out after 30.001s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc001030540\u003e: \n read access verification failed: get pods -n kube-system: pods is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"pods\" in API group \"\" in the namespace \"kube-system\"\n get configmaps -n kube-system: configmaps is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"configmaps\" in API group \"\" in the namespace \"kube-system\"\n {\n msg: \"read access verification failed: get pods -n kube-system: pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\\nget configmaps -n kube-system: configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.joinError | 0xc000fb1de8\u003e{\n errs: [\n \u003c*fmt.wrapError | 0xc001100b60\u003e{\n msg: \"get pods -n kube-system: pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.StatusError | 0xc001106b40\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"pods is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"pods\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n Reason: \"Forbidden\",\n Details: {Name: \"\", Group: \"\", Kind: \"pods\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n },\n },\n \u003c*fmt.wrapError | 0xc0010304e0\u003e{\n msg: \"get configmaps -n kube-system: configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n err: \u003c*errors.StatusError | 0xc001020780\u003e{\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"configmaps is forbidden: User \\\"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\\\" cannot list resource \\\"configmaps\\\" in API group \\\"\\\" in the namespace \\\"kube-system\\\"\",\n Reason: \"Forbidden\",\n Details: {Name: \"\", Group: \"\", Kind: \"configmaps\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n },\n },\n ],\n },\n }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3c6a69913a4e7e3ddde30c4ad0c3578c4319954e53163360fc7432ced31dda4a","failure_pattern":"read access verification failed: get pod \u003cpod\u003e kube-system: pods is forbidden: User \"system:sre-break-glass:\u003cuuid\u003e\" cannot list resource \"pods\" in API group \"\" in the namespace \"\u003cnamespace\u003e\"","search_query":"read access verification failed: get pods -n kube-system: pods is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"pods\" in API group \"\" in the namespace \"kube-system\""}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["3c6a69913a4e7e3ddde30c4ad0c3578c4319954e53163360fc7432ced31dda4a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037243447915581440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4607,"pr_state":"closed","pr_sha":"db17c8e13fefddc5d7923aa639d1a6106db776c8","final_merged_sha":"db17c8e13fefddc5d7923aa639d1a6106db776c8","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-26T19:00:53Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"ba8de21c3a1e57adeb598fb46658905b3fc5d7a6fc8c1af038117bf583b7f4a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037243447915581440","occurred_at":"2026-03-26T19:00:53Z","failed_at":"e2e","signature_id":"e4f4a6260725e00f82e45f708df90f1cac7ff54e1af9dce79b2a7eaf265e8be3","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:109]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000757760\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-subnet-reuse-cjvhd4\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-nsb4p6\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-nsb4p6/providers/Microsoft.Resources/deployments/customer-infra-cluster-subnet-reuse-cjvhd4/operationStatuses/08584270527184429378\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"BadRequest\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"CannotChangePrivateLinkConnectionOnPrivateEndpoint\\\",\\r\\n \\\"message\\\": \\\"Cannot change the private link connection on private endpoint /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-nsb4p6/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-nsb4p6/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-cluster-subnet-reuse-cjvhd4\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-nsb4p6\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-nsb4p6/providers/Microsoft.Resources/deployments/customer-infra-cluster-subnet-reuse-cjvhd4/operationStatuses/08584270527184429378\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"CannotChangePrivateLinkConnectionOnPrivateEndpoint\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Cannot change the private link connection on private endpoint /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-nsb4p6/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-nsb4p6/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": []\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000757740\u003e{\n msg: \"failed waiting for deployment \\\"customer-infra-cluster-subnet-reuse-cjvhd4\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-nsb4p6\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-nsb4p6/providers/Microsoft.Resources/deployments/customer-infra-cluster-subnet-reuse-cjvhd4/operationStatuses/08584270527184429378\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"CannotChangePrivateLinkConnectionOnPrivateEndpoint\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Cannot change the private link connection on private endpoint /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-nsb4p6/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-nsb4p6/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": []\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0008e8cc0\u003e{\n ErrorCode: \"DeploymentFailed\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:109]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000757760\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-subnet-reuse-cjvhd4\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-nsb4p6\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-nsb4p6/providers/Microsoft.Resources/deployments/customer-infra-cluster-subnet-reuse-cjvhd4/operationStatuses/08584270527184429378\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"BadRequest\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"CannotChangePrivateLinkConnectionOnPrivateEndpoint\\\",\\r\\n \\\"message\\\": \\\"Cannot change the private link connection on private endpoint /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-nsb4p6/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-nsb4p6/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-cluster-subnet-reuse-cjvhd4\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-nsb4p6\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-nsb4p6/providers/Microsoft.Resources/deployments/customer-infra-cluster-subnet-reuse-cjvhd4/operationStatuses/08584270527184429378\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"CannotChangePrivateLinkConnectionOnPrivateEndpoint\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Cannot change the private link connection on private endpoint /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-nsb4p6/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-nsb4p6/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": []\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000757740\u003e{\n msg: \"failed waiting for deployment \\\"customer-infra-cluster-subnet-reuse-cjvhd4\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-nsb4p6\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-nsb4p6/providers/Microsoft.Resources/deployments/customer-infra-cluster-subnet-reuse-cjvhd4/operationStatuses/08584270527184429378\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"CannotChangePrivateLinkConnectionOnPrivateEndpoint\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Cannot change the private link connection on private endpoint /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-nsb4p6/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-nsb4p6/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": []\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0008e8cc0\u003e{\n ErrorCode: \"DeploymentFailed\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2b89c266bf59ce593cd58ea2501b8bb1da0d398ef7d59e90d5c15a4bcd081933","failure_pattern":"ERROR CODE: DeploymentFailed; detail code CannotChangePrivateLinkConnectionOnPrivateEndpoint; detail message Cannot change the private link connection on private endpoint /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"fc9348df8acb049c967f84c4e0feff9f830d1a81a1c1b6ef9602d4d8fa4ea13f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037243447915581440","occurred_at":"2026-03-26T19:00:53Z","failed_at":"e2e","signature_id":"03315b27f932aa160b882106306608c7c208749a58c97c5deaab14e8258f9023","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:96]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0014d4940\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-basic-hcp-cluster2-2x54nn\" in resourcegroup=\"customer-rg-h9bxs7\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-h9bxs7/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster2-2x54nn/operationStatuses/08584270525680151780\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"BadRequest\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"CannotChangePrivateLinkConnectionOnPrivateEndpoint\\\",\\r\\n \\\"message\\\": \\\"Cannot change the private link connection on private endpoint /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-h9bxs7/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-h9bxs7/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-basic-hcp-cluster2-2x54nn\\\" in resourcegroup=\\\"customer-rg-h9bxs7\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-h9bxs7/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster2-2x54nn/operationStatuses/08584270525680151780\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"CannotChangePrivateLinkConnectionOnPrivateEndpoint\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Cannot change the private link connection on private endpoint /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-h9bxs7/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-h9bxs7/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": []\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0014d4920\u003e{\n msg: \"failed waiting for deployment \\\"customer-infra-basic-hcp-cluster2-2x54nn\\\" in resourcegroup=\\\"customer-rg-h9bxs7\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-h9bxs7/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster2-2x54nn/operationStatuses/08584270525680151780\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"CannotChangePrivateLinkConnectionOnPrivateEndpoint\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Cannot change the private link connection on private endpoint /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-h9bxs7/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-h9bxs7/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": []\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000d84cf0\u003e{\n ErrorCode: \"DeploymentFailed\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:96]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0014d4940\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-basic-hcp-cluster2-2x54nn\" in resourcegroup=\"customer-rg-h9bxs7\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-h9bxs7/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster2-2x54nn/operationStatuses/08584270525680151780\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"BadRequest\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"CannotChangePrivateLinkConnectionOnPrivateEndpoint\\\",\\r\\n \\\"message\\\": \\\"Cannot change the private link connection on private endpoint /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-h9bxs7/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-h9bxs7/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-basic-hcp-cluster2-2x54nn\\\" in resourcegroup=\\\"customer-rg-h9bxs7\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-h9bxs7/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster2-2x54nn/operationStatuses/08584270525680151780\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"CannotChangePrivateLinkConnectionOnPrivateEndpoint\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Cannot change the private link connection on private endpoint /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-h9bxs7/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-h9bxs7/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": []\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0014d4920\u003e{\n msg: \"failed waiting for deployment \\\"customer-infra-basic-hcp-cluster2-2x54nn\\\" in resourcegroup=\\\"customer-rg-h9bxs7\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-h9bxs7/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster2-2x54nn/operationStatuses/08584270525680151780\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"CannotChangePrivateLinkConnectionOnPrivateEndpoint\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Cannot change the private link connection on private endpoint /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-h9bxs7/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-h9bxs7/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": []\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000d84cf0\u003e{\n ErrorCode: \"DeploymentFailed\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2b89c266bf59ce593cd58ea2501b8bb1da0d398ef7d59e90d5c15a4bcd081933","failure_pattern":"ERROR CODE: DeploymentFailed; detail code CannotChangePrivateLinkConnectionOnPrivateEndpoint; detail message Cannot change the private link connection on private endpoint /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["2b89c266bf59ce593cd58ea2501b8bb1da0d398ef7d59e90d5c15a4bcd081933"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4625/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037239394653442048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4625,"pr_state":"closed","pr_sha":"949f619a3fae046c86d1bcdbb9d392fc4ae4c128","final_merged_sha":"d00be7a802af61e780e9a2815cffe5e799eba275","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-26T18:44:56Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"6dfb77bf859818c013402fc9e750d071f89bb41bfa1f613ca642bd4c088ad463","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4625/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037239394653442048","occurred_at":"2026-03-26T18:44:56Z","failed_at":"provision","signature_id":"38b96729aa55faad933fbf6565b47e698ba52a490a746835485fa51ff65fb48b","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-26T19:27:42.856Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-26T19:27:43.551Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:571b690d-4bcd-4d64-8d4e-a350c2693d67 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j3442048] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j3442048.hcpsvc.osadev.cloud] image:map[digest:sha256:7b5ae3d95bc764234f0b78cbc111ee7528b6efb8b2a9027e6e6a707fb66cb5b9 registry:arohcpsvcdev.azurecr.io repository:ci-op-br33ciqz/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-br33ciqz/pipeline:pull workloadIdentityClientId:442e39d9-a6b5-4cab-9f5d-38d223e0b76e workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:a3867303-90c6-46e9-b871-b2d7f79da93a workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-03-26T19:27:43.551Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-26T19:27:43.819Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy namespace=sessiongate\ntime=2026-03-26T19:27:44.123Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy dryRun=false\ntime=2026-03-26T19:27:44.123Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-26T19:27:44.236Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-26T19:27:51.065Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.6:15017, code 500: 500 Internal Server Error\"\ntime=2026-03-26T19:27:51.065Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-26T19:27:51.065Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-26T19:27:52.732Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy release=sessiongate namespace=sessiongate status=failed description=\"Release \\\"sessiongate\\\" failed: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.6:15017, code 500: 500 Internal Server Error\" values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:571b690d-4bcd-4d64-8d4e-a350c2693d67 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j3442048] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j3442048.hcpsvc.osadev.cloud] image:map[digest:sha256:7b5ae3d95bc764234f0b78cbc111ee7528b6efb8b2a9027e6e6a707fb66cb5b9 registry:arohcpsvcdev.azurecr.io repository:ci-op-br33ciqz/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-br33ciqz/pipeline:pull workloadIdentityClientId:442e39d9-a6b5-4cab-9f5d-38d223e0b76e workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:a3867303-90c6-46e9-b871-b2d7f79da93a workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-03-26T19:27:52.735Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5SUQU/cPhDF7/kUVk6stI5g//ypSsWBQqVKlaoVcKLqwXEeicHxpOMxCMSHr0LYEoVlF27OZOY3z+OnyTxEMSIltojqSFVGjJjSY4fKa1j54UJ1GIVdqOdqCP00Lf6FcIvwFImdsavw7FemlFL5OfjWWRxbSylIPld5RIyOQm0Erz6HmhOfooDPyE8z9PNZtyaYGi0G5KpwXYWlIEzeg9/otrlIe5gKrOFhxVHYAPnqQuVCvYFVrs34GGOiZwvzFJ2n+9Wcto/+zXxtweKunO3PjCtGbIaRmpuoXRRH2oWaEeOKdWx5mbwfXalL3m8RfA7LkCXTravAJ970vIkHbLex70mKQu3Zs6NPceWCmzxdLEbAwjDpxnaFeUiMwlI79tSzgd83vuMkDbF7MH2/JXln75+kek93U7EfBwSS5u3Jfb+4WJ5RWqNtvdvwJyFK3xBB+nd97W7NQ5I2SZopJ/v9JbtJJThAEL/1SyBmj+quAUOJaxHFtJ0qIXdAUDuVEfThnXyxuzjQu//pxcHF3ufDxafD/f3LfKaKQm3I+X/vMp/Nskd1TS6oGxeqIxcCeLS6KKiXjTVeVdMdlT2qjqn/+yJ0rjqqhuT3QebKUz1XLWI0dY8krsCqvB9dvkK0fwMAAP//DSzs0mAFAAA=\"\ntime=2026-03-26T19:27:52.737Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-26T19:27:52.737Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.6:15017, code 500: 500 Internal Server Error\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a83e14f56d71f4d9dfea9f79e4a2b2d28df34e80a7e1b1939e2b98b6de4b1e22","failure_pattern":"Internal error occurred: failed calling webhook \"validation.istio.io\": failed to call webhook: Post \"\u003curl\u003e\": proxy error from localhost:9443 while dialing \u003cip\u003e:\u003cport\u003e, code 500: 500 Internal Server Error","search_query":"error running Helm release deployment Step, failed to deploy helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": proxy error from localhost:9443 while dialing 10.128.64.6:15017, code 500: 500 Internal Server Error"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a83e14f56d71f4d9dfea9f79e4a2b2d28df34e80a7e1b1939e2b98b6de4b1e22"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4653/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037234236766818304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4653,"pr_state":"closed","pr_sha":"3963f2aed54f7786a885540614d69c6642229697","final_merged_sha":"3963f2aed54f7786a885540614d69c6642229697","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-26T18:24:26Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4652/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037230901053624320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4652,"pr_state":"closed","pr_sha":"efa0d84851e5101928b606673745523e80f3457b","final_merged_sha":"ef0f1b19b5b4a0415c36e782700ec263a51ca311","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-26T18:11:11Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"78ba15f3fb4ca2c0369bc5bf250daaa72eaf6a99ead18093ecdfbac4106c25c6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4652/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037230901053624320","occurred_at":"2026-03-26T18:11:11Z","failed_at":"e2e","signature_id":"908f10112c7eb2124eba5b63368ceea8a2921cfb611967b1a02b86c034b07d32","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:109]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000eae9e0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-subnet-reuse-n58pg4\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-wcrvrb\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-wcrvrb/providers/Microsoft.Resources/deployments/customer-infra-cluster-subnet-reuse-n58pg4/operationStatuses/08584270553481553506\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"BadRequest\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"CannotChangePrivateLinkConnectionOnPrivateEndpoint\\\",\\r\\n \\\"message\\\": \\\"Cannot change the private link connection on private endpoint /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-wcrvrb/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-wcrvrb/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-cluster-subnet-reuse-n58pg4\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-wcrvrb\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-wcrvrb/providers/Microsoft.Resources/deployments/customer-infra-cluster-subnet-reuse-n58pg4/operationStatuses/08584270553481553506\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"CannotChangePrivateLinkConnectionOnPrivateEndpoint\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Cannot change the private link connection on private endpoint /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-wcrvrb/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-wcrvrb/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": []\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000eae9c0\u003e{\n msg: \"failed waiting for deployment \\\"customer-infra-cluster-subnet-reuse-n58pg4\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-wcrvrb\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-wcrvrb/providers/Microsoft.Resources/deployments/customer-infra-cluster-subnet-reuse-n58pg4/operationStatuses/08584270553481553506\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"CannotChangePrivateLinkConnectionOnPrivateEndpoint\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Cannot change the private link connection on private endpoint /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-wcrvrb/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-wcrvrb/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": []\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0009d2210\u003e{\n ErrorCode: \"DeploymentFailed\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:109]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000eae9e0\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-subnet-reuse-n58pg4\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-wcrvrb\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-wcrvrb/providers/Microsoft.Resources/deployments/customer-infra-cluster-subnet-reuse-n58pg4/operationStatuses/08584270553481553506\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"BadRequest\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"CannotChangePrivateLinkConnectionOnPrivateEndpoint\\\",\\r\\n \\\"message\\\": \\\"Cannot change the private link connection on private endpoint /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-wcrvrb/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-wcrvrb/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-cluster-subnet-reuse-n58pg4\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-wcrvrb\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-wcrvrb/providers/Microsoft.Resources/deployments/customer-infra-cluster-subnet-reuse-n58pg4/operationStatuses/08584270553481553506\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"CannotChangePrivateLinkConnectionOnPrivateEndpoint\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Cannot change the private link connection on private endpoint /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-wcrvrb/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-wcrvrb/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": []\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000eae9c0\u003e{\n msg: \"failed waiting for deployment \\\"customer-infra-cluster-subnet-reuse-n58pg4\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-wcrvrb\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-wcrvrb/providers/Microsoft.Resources/deployments/customer-infra-cluster-subnet-reuse-n58pg4/operationStatuses/08584270553481553506\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"CannotChangePrivateLinkConnectionOnPrivateEndpoint\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Cannot change the private link connection on private endpoint /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-wcrvrb/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-wcrvrb/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": []\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0009d2210\u003e{\n ErrorCode: \"DeploymentFailed\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2b89c266bf59ce593cd58ea2501b8bb1da0d398ef7d59e90d5c15a4bcd081933","failure_pattern":"ERROR CODE: DeploymentFailed; detail code CannotChangePrivateLinkConnectionOnPrivateEndpoint; detail message Cannot change the private link connection on private endpoint /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"b0b13521e07e65cd0c91c7e46f458722e04f33fc45d0c474f7a9ab5e0fb3351e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4652/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037230901053624320","occurred_at":"2026-03-26T18:11:11Z","failed_at":"e2e","signature_id":"6bd86f2debe73adcbada62620c928abfc1f140f16a8dec7e5ffe577bf0825858","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:96]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000046020\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-basic-hcp-cluster2-qdjsd4\" in resourcegroup=\"customer-rg-nqljpg\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-nqljpg/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster2-qdjsd4/operationStatuses/08584270551842672141\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"BadRequest\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"CannotChangePrivateLinkConnectionOnPrivateEndpoint\\\",\\r\\n \\\"message\\\": \\\"Cannot change the private link connection on private endpoint /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-nqljpg/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-nqljpg/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-basic-hcp-cluster2-qdjsd4\\\" in resourcegroup=\\\"customer-rg-nqljpg\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-nqljpg/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster2-qdjsd4/operationStatuses/08584270551842672141\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"CannotChangePrivateLinkConnectionOnPrivateEndpoint\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Cannot change the private link connection on private endpoint /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-nqljpg/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-nqljpg/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": []\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000046000\u003e{\n msg: \"failed waiting for deployment \\\"customer-infra-basic-hcp-cluster2-qdjsd4\\\" in resourcegroup=\\\"customer-rg-nqljpg\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-nqljpg/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster2-qdjsd4/operationStatuses/08584270551842672141\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"CannotChangePrivateLinkConnectionOnPrivateEndpoint\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Cannot change the private link connection on private endpoint /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-nqljpg/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-nqljpg/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": []\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0010ba450\u003e{\n ErrorCode: \"DeploymentFailed\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:96]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000046020\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-basic-hcp-cluster2-qdjsd4\" in resourcegroup=\"customer-rg-nqljpg\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-nqljpg/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster2-qdjsd4/operationStatuses/08584270551842672141\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"BadRequest\",\n \"message\": \"{\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"CannotChangePrivateLinkConnectionOnPrivateEndpoint\\\",\\r\\n \\\"message\\\": \\\"Cannot change the private link connection on private endpoint /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-nqljpg/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-nqljpg/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-basic-hcp-cluster2-qdjsd4\\\" in resourcegroup=\\\"customer-rg-nqljpg\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-nqljpg/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster2-qdjsd4/operationStatuses/08584270551842672141\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"CannotChangePrivateLinkConnectionOnPrivateEndpoint\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Cannot change the private link connection on private endpoint /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-nqljpg/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-nqljpg/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": []\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000046000\u003e{\n msg: \"failed waiting for deployment \\\"customer-infra-basic-hcp-cluster2-qdjsd4\\\" in resourcegroup=\\\"customer-rg-nqljpg\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-nqljpg/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster2-qdjsd4/operationStatuses/08584270551842672141\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"BadRequest\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"CannotChangePrivateLinkConnectionOnPrivateEndpoint\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Cannot change the private link connection on private endpoint /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-nqljpg/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-nqljpg/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": []\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0010ba450\u003e{\n ErrorCode: \"DeploymentFailed\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2b89c266bf59ce593cd58ea2501b8bb1da0d398ef7d59e90d5c15a4bcd081933","failure_pattern":"ERROR CODE: DeploymentFailed; detail code CannotChangePrivateLinkConnectionOnPrivateEndpoint; detail message Cannot change the private link connection on private endpoint /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint. Please ensure you are not updating the details of existing private link connection: '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/privateEndpoints/kv-private-endpoint/privateLinkServiceConnections/kv-private-endpoint'. That is not allowed.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"d3111801aa2fd72dc270d5516a1198aa936aaca167345f1dbe5cf0d23eb99d5a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4652/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037230901053624320","occurred_at":"2026-03-26T18:11:11Z","failed_at":"e2e","signature_id":"58dcc6d8cad33c56493d4d47fb1555033ec8706960400c06eb97d69126276f35","test_name":"Customer should perform a control plane y-stream upgrade from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:164]: Unexpected error:\n \u003c*exported.ResponseError | 0xc0008f9ad0\u003e: \n PATCH https://rp.j3624320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-5pq6sw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-d4srhh\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Request-Id\": [\n \"7a22645f-db58-4a17-b26f-09787f0217a0\",\n ],\n \"Date\": [\n \"Thu, 26 Mar 2026 19:36:23 GMT\",\n ],\n \"Content-Length\": [\"188\"],\n \"X-Envoy-Upstream-Service-Time\": [\"65\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000ca4ec0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 82, 101, 113, 117, 105, 114, 101, 100, 32, 118, 97, 108, 117, 101, 58, 32, 102, 105, 101, 108, 100, 32, 99, 97, 110, 110, 111, 116, 32, 98, 101, 32, 110, 117, 108, 108, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 112, 114, 111, 112, 101, 114, 116, 105, 101, 115, 46, 112, 108, 97, 116, 102, 111, 114, 109, 46, 118, 110, 101, 116, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 83, 117, 98, 110, 101, 116, 73, 100, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 188,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PATCH\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j3624320.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-5pq6sw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-d4srhh\",\n RawPath: \"\",\n OmitHost: false,\n ForceQuery: false,\n RawQuery: \"api-version=2025-12-23-preview\",\n Fragment: \"\",\n RawFragment: \"\",\n },\n Proto: \"HTTP/1.1\",\n ProtoMajor: 1,\n ProtoMinor: 1,\n Header: {\n \"Authorization\": [\"redacted\"],\n \"X-Ms-Identity-Url\": [\n \"https://dummyhost.identity.azure.net\",\n ],\n \"Accept\": [\"application/json\"],\n \"Content-Length\": [\"66\"],\n \"Content-Type\": [\"application/json\"],\n \"User-Agent\": [\n \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7 (Red Hat 1.25.7-1.el9_7) X:strictfipsruntime; linux)\",\n ],\n \"X-Ms-Arm-Resource-System-Data\": [\n \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"createdAt\\\": \\\"2026-03-26T19:36:23Z\\\"}\",\n ],\n },\n Body: \u003c*http.readTrackingBody | 0xc000c8a7f8\u003e{\n ReadCloser: \u003c*runtime.retryableRequestBody | 0xc0007e9380\u003e{\n body: \u003cexported.nopCloser\u003e{\n ReadSeeker: \u003c*bytes.Reader | 0xc0004f7050\u003e{\n s: \"{\\\"properties\\\":{\\\"version\\\":{\\\"channelGroup\\\":\\\"stable\\\",\\\"id\\\":\\\"4.21.6\\\"}}}\",\n i: 66,\n prevRune: -1,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:164]: Unexpected error:\n \u003c*exported.ResponseError | 0xc0008f9ad0\u003e: \n PATCH https://rp.j3624320.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-5pq6sw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-d4srhh\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Request-Id\": [\n \"7a22645f-db58-4a17-b26f-09787f0217a0\",\n ],\n \"Date\": [\n \"Thu, 26 Mar 2026 19:36:23 GMT\",\n ],\n \"Content-Length\": [\"188\"],\n \"X-Envoy-Upstream-Service-Time\": [\"65\"],\n \"Server\": [\"istio-envoy\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000ca4ec0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 82, 101, 113, 117, 105, 114, 101, 100, 32, 118, 97, 108, 117, 101, 58, 32, 102, 105, 101, 108, 100, 32, 99, 97, 110, 110, 111, 116, 32, 98, 101, 32, 110, 117, 108, 108, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 112, 114, 111, 112, 101, 114, 116, 105, 101, 115, 46, 112, 108, 97, 116, 102, 111, 114, 109, 46, 118, 110, 101, 116, 73, 110, 116, 101, 103, 114, 97, 116, 105, 111, 110, 83, 117, 98, 110, 101, 116, 73, 100, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 188,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PATCH\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"rp.j3624320.hcpsvc.osadev.cloud\",\n Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-5pq6sw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-d4srhh\",\n RawPath: \"\",\n OmitHost: false,\n ForceQuery: false,\n RawQuery: \"api-version=2025-12-23-preview\",\n Fragment: \"\",\n RawFragment: \"\",\n },\n Proto: \"HTTP/1.1\",\n ProtoMajor: 1,\n ProtoMinor: 1,\n Header: {\n \"Authorization\": [\"redacted\"],\n \"X-Ms-Identity-Url\": [\n \"https://dummyhost.identity.azure.net\",\n ],\n \"Accept\": [\"application/json\"],\n \"Content-Length\": [\"66\"],\n \"Content-Type\": [\"application/json\"],\n \"User-Agent\": [\n \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7 (Red Hat 1.25.7-1.el9_7) X:strictfipsruntime; linux)\",\n ],\n \"X-Ms-Arm-Resource-System-Data\": [\n \"{\\\"createdBy\\\": \\\"e2e-test\\\", \\\"createdByType\\\": \\\"Application\\\", \\\"createdAt\\\": \\\"2026-03-26T19:36:23Z\\\"}\",\n ],\n },\n Body: \u003c*http.readTrackingBody | 0xc000c8a7f8\u003e{\n ReadCloser: \u003c*runtime.retryableRequestBody | 0xc0007e9380\u003e{\n body: \u003cexported.nopCloser\u003e{\n ReadSeeker: \u003c*bytes.Reader | 0xc0004f7050\u003e{\n s: \"{\\\"properties\\\":{\\\"version\\\":{\\\"channelGroup\\\":\\\"stable\\\",\\\"id\\\":\\\"4.21.6\\\"}}}\",\n i: 66,\n prevRune: -1,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1b131b2f9dd20d650a275cc9a5f957034289c13e33a1a1ca48259b4fd6d16bed","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["1b131b2f9dd20d650a275cc9a5f957034289c13e33a1a1ca48259b4fd6d16bed","2b89c266bf59ce593cd58ea2501b8bb1da0d398ef7d59e90d5c15a4bcd081933"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037224739394555904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4638,"pr_state":"closed","pr_sha":"c90b1b886828f94625c6676c30ff59cdc68531ea","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-26T17:46:37Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"1da2fe8157e93b90ba133145da03f40084eba0225eb4a52d1bf4f37addfc70c8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037224739394555904","occurred_at":"2026-03-26T17:46:37Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-03-26T18:16:27.487Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n Kind: ARM\\n Template: templates/region.bicep\\n Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-03-26T18:16:30.165Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-03-26T18:16:31.296Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=6370d42ed2cab467b9d8a9e641c179817ed19b04da53274e67464cff44bd32f4\ntime=2026-03-26T18:18:01.839Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j4555904/providers/Microsoft.Resources/deployments/6370d42ed2cab467b9d8a9e641c179817ed19b04da53274e67464cff44bd32f4/operationStatuses/08584270578948601026\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4555904/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j4555904/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j4555904\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 3e5d19b8-bdfb-4823-b567-c6d7077f42fb:3/26/2026 6:17:48 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037217072429142016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4004,"pr_state":"closed","pr_sha":"ebcd306115a8da93c9cd79ae5c43f353b892fddd","final_merged_sha":"9feee09ab6996cfe7dd433b262345799b7851593","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-26T17:16:14Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"8e2129d15238777067163cf2b9a656799a0cb26269ee141ed13e6652220a888e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037217072429142016","occurred_at":"2026-03-26T17:16:14Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4507/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037204374710652928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4507,"pr_state":"closed","pr_sha":"c8fe0327ed4d63831dde849164226db8e68d1934","final_merged_sha":"0e19405a0a977d3cd718004121a44142d3aebb42","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-26T16:25:46Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"cce3100e69a39548544fc41adea9130a3a56626e054c1745286a760f88cf1d64","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4507/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037204374710652928","occurred_at":"2026-03-26T16:25:46Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4578/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037201982778773504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4578,"pr_state":"closed","pr_sha":"2a7d72c1118b2f2efc82657f3164075ad43e8d3c","final_merged_sha":"35d72d570de220208509fd109c0b0be041d2858a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-26T16:16:16Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"82ea3cc2089f909a0148622447e894965ab981ada26352b107a00b24fff7501f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4578/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037201982778773504","occurred_at":"2026-03-26T16:16:16Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4507/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037195004438384640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4507,"pr_state":"closed","pr_sha":"c8fe0327ed4d63831dde849164226db8e68d1934","final_merged_sha":"0e19405a0a977d3cd718004121a44142d3aebb42","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-26T15:48:15Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ee862270d14db220b2f7386548ce6cb508db5fc993ec65559b596200a12184e6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4507/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037195004438384640","occurred_at":"2026-03-26T15:48:15Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4607/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037194825786200064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4607,"pr_state":"closed","pr_sha":"f62a3d0dca2d6f65a3216b9bc21373aca3ba9df1","final_merged_sha":"db17c8e13fefddc5d7923aa639d1a6106db776c8","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-26T15:47:44Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4507/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037194173571928064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4507,"pr_state":"closed","pr_sha":"84ba1cf928e83c04a673b161103f41adc55b2a9f","final_merged_sha":"0e19405a0a977d3cd718004121a44142d3aebb42","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-26T15:45:14Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"89a415f99c4369059166c0ed51afb292fb3c7cd753d4450b72bcf56d18da5a08","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4507/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037194173571928064","occurred_at":"2026-03-26T15:45:14Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4630/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037193695987503104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4630,"pr_state":"closed","pr_sha":"a9338f4d9dcd719df5d5909f37464f394f220f54","final_merged_sha":"7e07b7c9223871a196c30b61870b1a0f20ef1025","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-26T15:43:20Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037185376379998208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4638,"pr_state":"closed","pr_sha":"9ec597aece4069a6202bb3ce204d2145ce08418c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-26T15:10:17Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4651/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037182423397044224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4651,"pr_state":"closed","pr_sha":"6d025920a926606b0cf9684cc84a473cb8e69ee0","final_merged_sha":"6d025920a926606b0cf9684cc84a473cb8e69ee0","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-26T14:58:33Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4507/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037179620826025984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4507,"pr_state":"closed","pr_sha":"24f6bb8fffc5a13602aee373c8f93fe7e492ecf7","final_merged_sha":"0e19405a0a977d3cd718004121a44142d3aebb42","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-26T14:47:18Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"7f3a97aeb44bd12d0d0deeb40128bf8ee641b93376ee3b49cb4482edf03db3c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4507/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037179620826025984","occurred_at":"2026-03-26T14:47:18Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037178034976460800","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4645,"pr_state":"closed","pr_sha":"05c202fbec411bd3fac71185ee1084f14cf57df4","final_merged_sha":"c66c503c1b9d5df065741762f0079c31491b75e8","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-26T14:41:07Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4507/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037177836791402496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4507,"pr_state":"closed","pr_sha":"a512416f33265c73625b495e90882e7f5f637d3c","final_merged_sha":"0e19405a0a977d3cd718004121a44142d3aebb42","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-26T14:40:10Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"7954af532a15401c538adcf63a0bbff3de4c744ce2d2378138aa0fa31cda59fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4507/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037177836791402496","occurred_at":"2026-03-26T14:40:10Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037175802902089728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4640,"pr_state":"closed","pr_sha":"149d110be2591b0b588fdc08fae8930aaff205ae","final_merged_sha":"e23d7c6b5233dccc23b0d4b24f1d727a143ec398","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-26T14:32:14Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"bd1bec8bc4a83f105ba245cda36e827e8b57e83196e3dd7f1d171536520c7702","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037175802902089728","occurred_at":"2026-03-26T14:32:14Z","failed_at":"e2e","signature_id":"27a655fb2e39758f53bcd450b3ad6aa5123a13c87da1437533ef4e958004a61a","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:214]: Timed out after 1800.003s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00090ba80\u003e: \n node pool upgrade verification failed: q0f3h5j5u6h0w3l-npupgrade-4-20-kdc5m-k9xcn (version 4.20 not in same minor as expected 4.21.5); nodes=[{\"name\":\"q0f3h5j5u6h0w3l-npupgrade-4-20-bgvp5-gwb6f\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.34.5-3.rhaos4.21.gita8af6ea.el9\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:70dcc3c7d3a572ecacc7c838fd17d07ff21531e50a7ed9227a17c27437151bf4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f33338fd94d039dc87a8f9c0fa00221c31bc493bedb70075f21a0fc912bfef04\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:185a95b130b6a27ad88fcf39803215b7bd998472b1e37225858435f4eb7c3c04\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ab5375f4de81dcf4e990e316124f3d9614ee588bc55342fe94b544602883afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:51e1b5dc9fccbee5b7df78d0e8bbe379f75529232c102c9ed5b1ae134c4b9b4c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:89c8b051a435031bdcc24715379e0003cb67001076c4134f82ad0dae5cef3704\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6148e980bdc4394e5ee766bac607fc607777a5c1c2dad4c18e243c006121235c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d5547fcebf920ead6c1f429068078852ea910912bb1ac2942a5f05f8eea4e968\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:746e5cb3f7701cf0afcbea92f34ada0c4ea060fc2691a59ebd1ed8239a8e61b5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:abd3eaae48639c784a21da52d7da93bd9ba38c938fc5941c9990958fd6b47f5d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31e175b644c6627779ca6b9f48d9426760dd449788ad671dd0277cce8fb822e8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69e134b501fdb83fc21e3e9e28b43d0f9ddf6643e2a87fc952d2ab48739121a9\",\"arohcpocpdev.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0fd69333c121efd68919e731cd3ea01e976e00415bda6e987bf1e949ec224eda\",\"arohcpocpdev.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:af3a4a739d80ee0b0d8a6c24b0dd9206d3e91907593da7cad54d881306b287ad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3da5a449ace61cb7d002d8e7911c14905c74cf58a296527458d1af0f1abf0a72\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:464c7d227e1c80e413d87d99fdf42fc80d76b3bc8f9281a23b7d59b3b4ea2372\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6ea0f19030ffc52a290e6eeb812ef2426c48a2f1083b1e415e0667998cbb7bf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cf9f6b6fecd35f94d7cdb57845990602be060e4b548aac8ab6bc1c96e22f9bb2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1c5b548d67e5c56e3297b2f8aa34428e544b5b5ac88bd8d0f465760375c501e6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bc7e9750e4647b2ccd9a7c1c0f4cc35f640b36d8ab9d3fa789eb2764afce3d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6b085cdad58db8529d99c22ec1ec79fe09ab00ab1c92ee38b29e7875dc0533c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e2efd3418ad3517f5db50de37cda00d27bb568554e7cb618ade7c8eebfbc9185\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:24929612f2f0f2d4cd3ac163c04bbf4ac64cc47d3d0eef81db085d00c59058bd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:987fe55afa2ee263467ecaf5c058664ad8e55126687f79d28ab5fa030ec17768\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cca646f56344329bd71967e6b044f43543f52f8c6a8986d725610ad92df26989\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d0d866dee27d9169a1e4711a35a4dfbc3ffe9655258a72188282f4c32eaa84a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2ade6dfb5406ef3a4264dea95d455298f54e8ef4d458d9ee3e94c33d48676a9b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6ba2d747627cdfa2d7d36728dac5444865455d422df60f33943a45cef211badb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b9f60077c475f2baad0dd51ffc9d469f3a9b54c474bb86a35f93cb9b8fdc734\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bccd3f544da031a055000bcf0a8fa1003dbe9cfa898bf2fb67c4ee3dcb4f4acc\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:17e68da37c74bdc95a4eb43b51ce671027807ee610a4ee5237324a8c0dc9d044\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b48b717cea29d8445ea717563e3794b381f6c1545b4458fb915c8baaa279893a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:12aec97ddeb527881acf5949bea2b227baa1dbfccf30679e8d57ff0b82e42707\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f88c407e2bdf86c7339ced2ae92bd716d5f74cee0999d41c5e3b26d47cfe2648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a26a47c35b590b15b04a9558c4fe3e34eef0fecfb4ab6e88274b7e9d70678a8d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c7f80064deb88df54bf40b4834fc5ffa9cd85fc18987fc82c0efdf579e228692\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2044d490f2d73d42bb6a114a2011c898034354e0fab1a872a723174b2c8327cc\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39a8fe4ceb03e1379951609b8d1ab33c40aa19f7fe24ed7b0f1621950c051cf6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:05ebc4de717d2d36e3d7daca025294e6de1181e3c67ebc38028fc0b0ea5bfafa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:23c2d9d4c3f769427c52b6b016633476fcf07a25523a0a3c6bcbf2a1b4840826\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20a3e59566dd25058fff22e18569067fe6a12d8ac5ac62fa6508e0cee44d7897\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a405e40da35645d318b95c414f68536e3d3e2db22394ca9bf9738e666336e462\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:12448aad6e9050c2b2445035be818a490cef502d39b73dbd51cc0f5ea7d67d40\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:17a5d6b121e2bb5b8656a7a427824c2af9254ee0bd1f5cec30072b7c564eae7f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4c39dc633bf3560c2397281b74faf9d00500b0738dec6c1aad2cc8ce326be6fc\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba77b4a636a254ef48caadc41666eb92b785317472708cf53d4a108cf7af9a78\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5b51a4d9d3b7ecad110fb6eb00f71e9e9b854067d79b220df397a58d9fa50eb0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9fe72e4ea5c3ef2600e9005c2ee869f9fbb68e31b569195e351a55ede1df703a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92a731e25801365be0ad0916bec5bf24b2035e443ad1414a1631d1e0473ddf77\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c91b7e9bdf73069bea2bc8ece90ce15c2a91a2bf99d9d970732b9afa233f3c6c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3f7181447dd3d28779323dac4561ca98d33755c19f468fb181d50c5338eca42a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:46427cb447a096a86618be4829484f964f10c220cd4d779e2c54a960c3045b7b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7e0a58aad44d5b09954f8261e3e428c8dfa076eb9c2b6ef46a91d27ac78ac17a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:83344c7d1374cbc2607b4fcb837f094c035b32daf24f8b3ccba2c8df409a6d3c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2f26472542da4f89a2ac34a93fd438d385419dda16811bdfa7e679e0115abe56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e19f841ff1628bef075f117708faad18b1f525e6629542bb2db3031acac948fe\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3739c0c55f7f7ccc84309ae11eb23d3d9e5ad50a68e849a8a8dfda7f2f2bc137\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a20df9dd8819885aa431022c6f5e80b77b4431f268b719d71bec26f10521da0e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6263faa8ef7759ec21f26df08314c5d64e52d967b49961cb9404c54532c03365\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b37b3692dc74deb0b12b592173a09fb371a0a87210861f14239c70f247431294\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:72ae38682f25786aec18625cfed5d7831017d2f0f1304c3320c00ce0b7fb09c1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9540d66e9824b1ba081ad80bb6715e5ccf61b42c29c1ad961c1863efd81f48ca\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:14e983ca10e1033822e1c4be0a1e4a40762951a5d6e4a77bf642891f291d32ae\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:698e13ede97d3e2fbc3d69858b3f2d5985bbdfb3fb0f646452fe0edabbd5eb7f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:273930332d7b481bbca4521cf740507498a2a323a4010cd66e896806b299f85e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f942448d1f3d05bd5534d16433e0961509c35ea1eab6d51c0d2a8b7e16b0ad29\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1e6d59867cbd750672bc9ccfb9c69c3d198174cd43dd6d71a0da358b4b766473\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e99cd629414427a7792ccecda36d856dbe549682438f7b0dfb0ee2e7af536dd6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44ceafe5147680d58a5c2d3720932b972e6bc9639faadc19aa6b360af154e1a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5b3d41bea99611b73a21042f0f416dd3e8293c93086e18193c98b5fc1994734e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c59765325e6f286c067e555850b0d81f2de5e5ee8625ed83cdc2cea96f3a7ddb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cdd07910c008557d42ada95c4e41333704779166ffa8397b0adc73050a2df3ab\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1b58ba57c10e2ba03a0d8dba1f3f8faa866e1dca6170725b6909c2cc2b96efa6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f02f34239d2898fd346b775194a7e3f1a26b8aaeeaf3670b67cc5e4021742420\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e476b4dafff885727b0057259c9df67a68e196f8cd92c21db2e4efe84a5f7a25\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f6ee6b4bbee84922afa10d46091e1511788478855b20ce2ad7ac803852f06694\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ad039a130515504a258952cb18e67c3329da5182bb0623cc0891e6c15e2113a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c1f86751c05c53394f4fa5375c71b390f0908ea4a0a8dcc1be27d1aaad2b8130\",\"arohcpocpdev.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4e62a2dcb8214491388d4c17f89fe699eb53cdb8039dcba6ab052edc28da3fbd\",\"arohcpocpdev.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:eea3f13f48a70c63b12c74b82caa6b1a8869130a43d5dc3c5ce7e518a4bdf4a6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cfcb0bd30c537e677ac6856c36c0c923b415f24d944c7aa1386489f511f6491d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f660d525da7c598367787fef3ac27294357524affe4f35c1f552e7f428ca7dab\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3e8d51b4b12d970bf561f67d0fcd9c0966a6d49866e6466daa341ee43d95d6e2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dadfaf3cceca2867f77c40b373b6fc80e34b4ba2cb1f05e7178a0cc79eac2d9d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:aa65d1c7fb8805b767fc4878faea280472f372010efadc66e5641a2e634a80c0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fcd01cbde0c58ad690f233a8f9ac9d080ca607fd7c24ad0de5afca564f098687\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:87de118dc29ca1b47d4768ec5ed4441bc1dada3fb2a78d53b1833249baa86dd4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b615eeec97d8429f11b1f8f9d05c7192c01ad293ed26c83b60bf6e31505f2c1f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64dbe77583c8b23dc6852fc482ea5b4081490734944ec8b765d5450b3fdb30ed\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9be8f12e3745100401dcf9f7d7e571260b11399f1dd98501208af756d08abd97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2e3661304917ce7f3be49974faacc60c24cbe2833d7b50c8c205c2f3f2467950\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d2afd0719d1a256d032048beeba2700cfe60da2904b2d9fe9c0aae3daf4318d5\",\"quay.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:1af59b7a29432314bde54e8977fa45fa92dc48885efbf0df601418ec0912f472\",\"quay.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:969bdfb00b7b9f7a99eb3892e34622055e6c98f72c74c64c9a60e1b4b881a246\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ad0e262327997e8cdc4074a2373e67274d9b8a943aa274a2cd8ce98ffe1db4a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d6c8c086bd9f73d1cc4a362ce6253914d4f4d8ce009b68aec42db9ff3a6aad36\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e5fa2595714628c16cf590efd28715bde790d2dc6b3a793b2a38dfb3ffb591a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5acacbb3eaaddf8d2f25e062427edd7f7636fc7a3b6a9fc84519df16fec0e5e0\"]},{\"name\":\"q0f3h5j5u6h0w3l-npupgrade-4-20-bgvp5-zf4gf\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.34.5-3.rhaos4.21.gita8af6ea.el9\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:70dcc3c7d3a572ecacc7c838fd17d07ff21531e50a7ed9227a17c27437151bf4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f33338fd94d039dc87a8f9c0fa00221c31bc493bedb70075f21a0fc912bfef04\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:185a95b130b6a27ad88fcf39803215b7bd998472b1e37225858435f4eb7c3c04\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ab5375f4de81dcf4e990e316124f3d9614ee588bc55342fe94b544602883afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:51e1b5dc9fccbee5b7df78d0e8bbe379f75529232c102c9ed5b1ae134c4b9b4c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:89c8b051a435031bdcc24715379e0003cb67001076c4134f82ad0dae5cef3704\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6148e980bdc4394e5ee766bac607fc607777a5c1c2dad4c18e243c006121235c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d5547fcebf920ead6c1f429068078852ea910912bb1ac2942a5f05f8eea4e968\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:746e5cb3f7701cf0afcbea92f34ada0c4ea060fc2691a59ebd1ed8239a8e61b5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:abd3eaae48639c784a21da52d7da93bd9ba38c938fc5941c9990958fd6b47f5d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31e175b644c6627779ca6b9f48d9426760dd449788ad671dd0277cce8fb822e8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69e134b501fdb83fc21e3e9e28b43d0f9ddf6643e2a87fc952d2ab48739121a9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3da5a449ace61cb7d002d8e7911c14905c74cf58a296527458d1af0f1abf0a72\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:464c7d227e1c80e413d87d99fdf42fc80d76b3bc8f9281a23b7d59b3b4ea2372\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6ea0f19030ffc52a290e6eeb812ef2426c48a2f1083b1e415e0667998cbb7bf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cf9f6b6fecd35f94d7cdb57845990602be060e4b548aac8ab6bc1c96e22f9bb2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1c5b548d67e5c56e3297b2f8aa34428e544b5b5ac88bd8d0f465760375c501e6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bc7e9750e4647b2ccd9a7c1c0f4cc35f640b36d8ab9d3fa789eb2764afce3d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6b085cdad58db8529d99c22ec1ec79fe09ab00ab1c92ee38b29e7875dc0533c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e2efd3418ad3517f5db50de37cda00d27bb568554e7cb618ade7c8eebfbc9185\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:24929612f2f0f2d4cd3ac163c04bbf4ac64cc47d3d0eef81db085d00c59058bd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:987fe55afa2ee263467ecaf5c058664ad8e55126687f79d28ab5fa030ec17768\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cca646f56344329bd71967e6b044f43543f52f8c6a8986d725610ad92df26989\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d0d866dee27d9169a1e4711a35a4dfbc3ffe9655258a72188282f4c32eaa84a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2ade6dfb5406ef3a4264dea95d455298f54e8ef4d458d9ee3e94c33d48676a9b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6ba2d747627cdfa2d7d36728dac5444865455d422df60f33943a45cef211badb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b9f60077c475f2baad0dd51ffc9d469f3a9b54c474bb86a35f93cb9b8fdc734\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bccd3f544da031a055000bcf0a8fa1003dbe9cfa898bf2fb67c4ee3dcb4f4acc\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:17e68da37c74bdc95a4eb43b51ce671027807ee610a4ee5237324a8c0dc9d044\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b48b717cea29d8445ea717563e3794b381f6c1545b4458fb915c8baaa279893a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:12aec97ddeb527881acf5949bea2b227baa1dbfccf30679e8d57ff0b82e42707\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f88c407e2bdf86c7339ced2ae92bd716d5f74cee0999d41c5e3b26d47cfe2648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a26a47c35b590b15b04a9558c4fe3e34eef0fecfb4ab6e88274b7e9d70678a8d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c7f80064deb88df54bf40b4834fc5ffa9cd85fc18987fc82c0efdf579e228692\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2044d490f2d73d42bb6a114a2011c898034354e0fab1a872a723174b2c8327cc\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39a8fe4ceb03e1379951609b8d1ab33c40aa19f7fe24ed7b0f1621950c051cf6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:05ebc4de717d2d36e3d7daca025294e6de1181e3c67ebc38028fc0b0ea5bfafa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:23c2d9d4c3f769427c52b6b016633476fcf07a25523a0a3c6bcbf2a1b4840826\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5b51a4d9d3b7ecad110fb6eb00f71e9e9b854067d79b220df397a58d9fa50eb0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9fe72e4ea5c3ef2600e9005c2ee869f9fbb68e31b569195e351a55ede1df703a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:66640a3f7bd65090345552611701a73e74bc44897b9d29e43b11f73b96598ac0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:adcbb731ad586feb2c21dbc4993b26a564bbcb26efffb3f94f9da38e27002ea8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92a731e25801365be0ad0916bec5bf24b2035e443ad1414a1631d1e0473ddf77\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c91b7e9bdf73069bea2bc8ece90ce15c2a91a2bf99d9d970732b9afa233f3c6c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3f7181447dd3d28779323dac4561ca98d33755c19f468fb181d50c5338eca42a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:46427cb447a096a86618be4829484f964f10c220cd4d779e2c54a960c3045b7b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7e0a58aad44d5b09954f8261e3e428c8dfa076eb9c2b6ef46a91d27ac78ac17a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:83344c7d1374cbc2607b4fcb837f094c035b32daf24f8b3ccba2c8df409a6d3c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3739c0c55f7f7ccc84309ae11eb23d3d9e5ad50a68e849a8a8dfda7f2f2bc137\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a20df9dd8819885aa431022c6f5e80b77b4431f268b719d71bec26f10521da0e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:72ae38682f25786aec18625cfed5d7831017d2f0f1304c3320c00ce0b7fb09c1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9540d66e9824b1ba081ad80bb6715e5ccf61b42c29c1ad961c1863efd81f48ca\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:14e983ca10e1033822e1c4be0a1e4a40762951a5d6e4a77bf642891f291d32ae\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:698e13ede97d3e2fbc3d69858b3f2d5985bbdfb3fb0f646452fe0edabbd5eb7f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:273930332d7b481bbca4521cf740507498a2a323a4010cd66e896806b299f85e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f942448d1f3d05bd5534d16433e0961509c35ea1eab6d51c0d2a8b7e16b0ad29\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1e6d59867cbd750672bc9ccfb9c69c3d198174cd43dd6d71a0da358b4b766473\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e99cd629414427a7792ccecda36d856dbe549682438f7b0dfb0ee2e7af536dd6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44ceafe5147680d58a5c2d3720932b972e6bc9639faadc19aa6b360af154e1a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5b3d41bea99611b73a21042f0f416dd3e8293c93086e18193c98b5fc1994734e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5bd409efe77c6a126418be2c098dde9431c1a290cec885a7e37c0c8ddc2337a8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c24f5d0b962f9b381df0f8d90d4c31616a95623fbca70de0a2da621aba6bd19f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2a07c1b2ef06bbfa3a491b84f7410e24de7cb487f49bbbb3bfd7cb3510c8429a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bd3da568f95c65b04a03d96d175d64efcabcb6fe9a454a1658bbfa243b16a0a7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c59765325e6f286c067e555850b0d81f2de5e5ee8625ed83cdc2cea96f3a7ddb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cdd07910c008557d42ada95c4e41333704779166ffa8397b0adc73050a2df3ab\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e476b4dafff885727b0057259c9df67a68e196f8cd92c21db2e4efe84a5f7a25\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f6ee6b4bbee84922afa10d46091e1511788478855b20ce2ad7ac803852f06694\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ad039a130515504a258952cb18e67c3329da5182bb0623cc0891e6c15e2113a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c1f86751c05c53394f4fa5375c71b390f0908ea4a0a8dcc1be27d1aaad2b8130\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cfcb0bd30c537e677ac6856c36c0c923b415f24d944c7aa1386489f511f6491d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f660d525da7c598367787fef3ac27294357524affe4f35c1f552e7f428ca7dab\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3e8d51b4b12d970bf561f67d0fcd9c0966a6d49866e6466daa341ee43d95d6e2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dadfaf3cceca2867f77c40b373b6fc80e34b4ba2cb1f05e7178a0cc79eac2d9d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5f852f6127d27e6fc071afe58d80024370467cb2476f4d9f077f8ebca8600918\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d6dd7872d95388063d6606823e4c4340f83a4f6983034f265a4d749f0418e77c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:aa65d1c7fb8805b767fc4878faea280472f372010efadc66e5641a2e634a80c0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fcd01cbde0c58ad690f233a8f9ac9d080ca607fd7c24ad0de5afca564f098687\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:87de118dc29ca1b47d4768ec5ed4441bc1dada3fb2a78d53b1833249baa86dd4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b615eeec97d8429f11b1f8f9d05c7192c01ad293ed26c83b60bf6e31505f2c1f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64dbe77583c8b23dc6852fc482ea5b4081490734944ec8b765d5450b3fdb30ed\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9be8f12e3745100401dcf9f7d7e571260b11399f1dd98501208af756d08abd97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2e3661304917ce7f3be49974faacc60c24cbe2833d7b50c8c205c2f3f2467950\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d2afd0719d1a256d032048beeba2700cfe60da2904b2d9fe9c0aae3daf4318d5\",\"quay.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:1af59b7a29432314bde54e8977fa45fa92dc48885efbf0df601418ec0912f472\",\"quay.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:969bdfb00b7b9f7a99eb3892e34622055e6c98f72c74c64c9a60e1b4b881a246\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ad0e262327997e8cdc4074a2373e67274d9b8a943aa274a2cd8ce98ffe1db4a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d6c8c086bd9f73d1cc4a362ce6253914d4f4d8ce009b68aec42db9ff3a6aad36\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e5fa2595714628c16cf590efd28715bde790d2dc6b3a793b2a38dfb3ffb591a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5acacbb3eaaddf8d2f25e062427edd7f7636fc7a3b6a9fc84519df16fec0e5e0\"]},{\"name\":\"q0f3h5j5u6h0w3l-npupgrade-4-20-kdc5m-k9xcn\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.33.9-3.rhaos4.20.gitb9ac835.el9\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:670004bffddb41c787b9f9a5c7d1615453d4acfa6452dedfa61ad2f305eea814\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:779115a9d5f9b0373950fdf99b8bb7bffb8a9143ec28c6e1558d1d3d8eec50e4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:70dcc3c7d3a572ecacc7c838fd17d07ff21531e50a7ed9227a17c27437151bf4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f33338fd94d039dc87a8f9c0fa00221c31bc493bedb70075f21a0fc912bfef04\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:185a95b130b6a27ad88fcf39803215b7bd998472b1e37225858435f4eb7c3c04\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ab5375f4de81dcf4e990e316124f3d9614ee588bc55342fe94b544602883afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3e262d5207d5e2077e4a14f4fbddc70d4d2651dbaaf955147013feb58a18cddd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8e08a29d3b206379957f1fb668058b4effe2f4542c2b4e45a3cfafa4b9a728e5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:862d5befeba504d5058264cf65f16fc307eea4eb3914fbbbf40fbe61714b0958\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9aee9be2da68239a2a559e3ca1211eb8988a7f83a4ff82b9bd7c21a436be9c22\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:51e1b5dc9fccbee5b7df78d0e8bbe379f75529232c102c9ed5b1ae134c4b9b4c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:89c8b051a435031bdcc24715379e0003cb67001076c4134f82ad0dae5cef3704\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ecf24d093581fc39574dceec3d50efd4c3146cebf49bb7a5a450b3f8ce740551\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fcdb0afaa38bf8deddfcda9abb50886c2a72c54695992d79c15a8413ac2d1796\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0cff1d43f1ec9e919965e5cedd2268233c0f7b27b07750bd6fe8a9bf10f65bf4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bf18d37657cff26bb88be6fbbdaf64ae8f1f090312233b0c50a904a35729c74c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ea1ff613255c1ee67b34ec82b68ec6f5a4a6d283620a8e0ad142d4a31eb9211\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a45c25e5322761a9268bbaf5bdf3436d4623f42b90cd15942df0e5c22755e3ac\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:04c7a2673571e9069c934d86b4308c0f57b2715b45db500837b7708ae1f20ae7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:23016e42a746a81e3de1f5661cf2673efbe15ad0926b92d3d7c6fa9ac47f7215\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7dd9584d291967387f2a1ade991ad3bf8565bb59f49b013217951399e8273498\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f6ef619d31c4211120b23a7bf3118c9db7329c81b94709df86396d269d6b30e6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:03a7d45ceb838c746bdcdac1a5cc02d26273b319d9da4498bb3ee31c3e130b43\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be6ed9469bb1acf2658aaf55b812e5334e308b4169b858b0bacf54ce6f4e2ca7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0c90ca6acbe2f4fdbd253ae22f5b3286247c6940321f7d2b9801aea21ef5269f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:26e4f5b039a340bc1ae080fb9d92799700b57ae5df45120768f516682d6cc54b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1adc4e7a9f123529e9df17eb1316a667d86ed2f185965c25ed6d32c44563188b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b963e4b54323ed252d951c924567590ccb4514d8e90b908c0be89f18919ae8cc\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31e175b644c6627779ca6b9f48d9426760dd449788ad671dd0277cce8fb822e8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69e134b501fdb83fc21e3e9e28b43d0f9ddf6643e2a87fc952d2ab48739121a9\",\"arohcpocpdev.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0fd69333c121efd68919e731cd3ea01e976e00415bda6e987bf1e949ec224eda\",\"arohcpocpdev.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:af3a4a739d80ee0b0d8a6c24b0dd9206d3e91907593da7cad54d881306b287ad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3da5a449ace61cb7d002d8e7911c14905c74cf58a296527458d1af0f1abf0a72\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:464c7d227e1c80e413d87d99fdf42fc80d76b3bc8f9281a23b7d59b3b4ea2372\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1652a6b3078719d56385dddda165a7e63168c645aa45ca408e7b12e682dcbdab\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7b0e43c637df9024af6222a689d54ba25e55f3a2a663202636b1b137b5a47a8c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6ea0f19030ffc52a290e6eeb812ef2426c48a2f1083b1e415e0667998cbb7bf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cf9f6b6fecd35f94d7cdb57845990602be060e4b548aac8ab6bc1c96e22f9bb2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:db40a3d4e50016c645fdb4a1f389dfcdbc9febc10c174816d681abe5c14153cd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f9b87244518245c63d1982182abe22775a2e207511cb328bc5c7b48e435b8da7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1c5b548d67e5c56e3297b2f8aa34428e544b5b5ac88bd8d0f465760375c501e6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bc7e9750e4647b2ccd9a7c1c0f4cc35f640b36d8ab9d3fa789eb2764afce3d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:04ee35aa6f87e954d386228808017cf2938c5a96a052059c6767078d5f3ad8d6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:16dfb8b96a665be805bc1000677823aee946d93bfff0d3c567f2a24ea28a537a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6b085cdad58db8529d99c22ec1ec79fe09ab00ab1c92ee38b29e7875dc0533c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e2efd3418ad3517f5db50de37cda00d27bb568554e7cb618ade7c8eebfbc9185\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2a9f74fb0da7fb9f7d70c01953cb70fdda6a22bb9182bc5358f5fd6d957f4f6f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ef42cb417552fe1e38eedc30e92f55c8c50aae1aa1b0b7094ab56118efe0d96c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2a4f6a54cd5135c8121832236063419071fa5d00c10a051c197d3d01370e4d9c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5dc767cd6267e006720af8cef541e23bc64d94ef9b1bf3a8b44b1fda5d9d8871\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:05376e7e2304d22dd866d8e25dd1e5ff8988b2232059fadfd12cb440af785635\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d6cf0a0701c4429a00483da429c3fa7d0f175d8e1602b2413860e8fc0c9f0e4f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9dee9b6866db30456c277ff1f7a114dd4ce25bff8aced70218623af9423ec3cf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3c906f4908c692697ae58891ef463ee62a23d4356bf00786947cc1dc02e4d53\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:78c23f5bccb6e82ade8428c8886ddc0291faec496590b2e6b5375ac6c0476dc3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9ff94628410a702de0cb3e1a715917829b36c3def524dcc0850ac34a811f8f6f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:876fb57ffb255ed75a452405bcd1d116788a34b19264886e6e087ff544c17083\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd6a017092d57ab3a6663f3822a0fa0e3bce4365859d9522dd173405276cd49a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ff360b712000d39cbb58158186ff0a18764278e6fa98ef9f21d113b978d9a79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:77d8ddf2957025a39616328b5cf2b012de6cedce63fb79eebc5270d7b3401574\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:11a864539ae77a94afcf8d2e1abf9168081c5081a1bc06088f4be5206cdc02df\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2bcca887297fc1168fa43a6c1acb7077bf198258fe2e0ef9379e11c418919c15\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bbb5af28bbed69e8189825fdc6665f4e8dbcd0c0c83fa479943ea102fe9452d6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c85dd52f4140690eef0d762834ab8b9af2faedbbd3bd9dd5616d4cd0bebd4740\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:124f1b391b0f376a5deffd05b29ef0896ca0912e1f67b3a2051ed6678c54a57e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ff8a8892f37210c582b3e526cbd2652b02ef9007d57be12b404434374dd0116a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:24929612f2f0f2d4cd3ac163c04bbf4ac64cc47d3d0eef81db085d00c59058bd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:987fe55afa2ee263467ecaf5c058664ad8e55126687f79d28ab5fa030ec17768\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cca646f56344329bd71967e6b044f43543f52f8c6a8986d725610ad92df26989\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d0d866dee27d9169a1e4711a35a4dfbc3ffe9655258a72188282f4c32eaa84a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2ade6dfb5406ef3a4264dea95d455298f54e8ef4d458d9ee3e94c33d48676a9b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6ba2d747627cdfa2d7d36728dac5444865455d422df60f33943a45cef211badb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:317131706dcce9c87eb4e1efb8446bdbf5e0726e120a8f6c655e2acd89d60e2d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ec94607aa5fce47591b1ed1939e86a0e1c43cc8ff439f42662c9ed32e414595e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b9f60077c475f2baad0dd51ffc9d469f3a9b54c474bb86a35f93cb9b8fdc734\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bccd3f544da031a055000bcf0a8fa1003dbe9cfa898bf2fb67c4ee3dcb4f4acc\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2962c6c31d7078b34cf7ba0c9b0ea685dc539cea67b06bd38b6ac566298bb790\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:321163a14934467a86e358bbb3115fbe104ad814bcaf7423f259eb82150da703\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5efa227ddac82e56eddc03761e220af1f2d6a4b22ee00b2baf426523f26c0319\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7376a666810d810549b4e270d9a2fd6bcfd8799370ff52476a4f799cd4e287d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:602dc9f92b7a8e84e6eb998191263d186d1d10d378eeef0845435715bee812dc\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f7953a8e77c5bb0efaa670ba2188050740eba0e1f6979248832c05cb886fa5d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:17e68da37c74bdc95a4eb43b51ce671027807ee610a4ee5237324a8c0dc9d044\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b48b717cea29d8445ea717563e3794b381f6c1545b4458fb915c8baaa279893a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7f426776d3ef70b585018b8d813f5eedd70373536681c6240df224ebf9ff6329\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d2906e0869191da12130ed79a57221aae735ef026d0f909534c7cbfbca131128\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3d68a9523f4c5c22a48dfa151bbfde78b2f360697afe3b9d8bfb53e8c359bf95\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7983b528aa73931fdaa817858ecd416dea0ec5a6917bebf5bd11b4eb83c752c9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8f13a1e673c882ee926e97270249a20d3c40797a8c7c5abb2fda9c98c24a2eb4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d4067116d68c1aabb5ee21c4addce05f6849f104d7a7f87f5f1fcef7990a3aee\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2af652f68b17ed49c2565928e4c101880e949c4484c6ee958793a8e668e47bea\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e199029f436d7247195d0c5b22909cb874ac67bd87d84c597a83ef7bd0d87ab0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:25d9148ea34149935323231c64952679727b6f7f235febc3be1df63e0bf18c5e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c4fdcf25b69610f918eac662369ed457ee400b3d5483e8fcbecd44a97c220302\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:12aec97ddeb527881acf5949bea2b227baa1dbfccf30679e8d57ff0b82e42707\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f88c407e2bdf86c7339ced2ae92bd716d5f74cee0999d41c5e3b26d47cfe2648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:41e0dcd2f91df9234249f168b3fbe6d55cfdb3ebd2e598d5edfa47f905c046cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9998ff90e081aac492498314bd02225cc3a8636d8c714ce8b618f4ee1de28c21\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:30a054cfe1b8330bd57d762921ca5cd281ad2fdd7994fd3093ffadb5d767a564\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9b369532d212af90d1232bcf0a84bc2616106733f4eae8d4aaf69fad595e1d67\"]}]\n {\n s: \"node pool upgrade verification failed: q0f3h5j5u6h0w3l-npupgrade-4-20-kdc5m-k9xcn (version 4.20 not in same minor as expected 4.21.5); nodes=[{\\\"name\\\":\\\"q0f3h5j5u6h0w3l-npupgrade-4-20-bgvp5-gwb6f\\\",\\\"ready\\\":true,\\\"containerRuntimeVersion\\\":\\\"cri-o://1.34.5-3.rhaos4.21.gita8af6ea.el9\\\",\\\"releaseImages\\\":[\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:70dcc3c7d3a572ecacc7c838fd17d07ff21531e50a7ed9227a17c27437151bf4\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f33338fd94d039dc87a8f9c0fa00221c31bc493bedb70075f21a0fc912bfef04\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:185a95b130b6a27ad88fcf39803215b7bd998472b1e37225858435f4eb7c3c04\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ab5375f4de81dcf4e990e316124f3d9614ee588bc55342fe94b544602883afa\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:51e1b5dc9fccbee5b7df78d0e8bbe379f75529232c102c9ed5b1ae134c4b9b4c\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:89c8b051a435031bdcc24715379e0003cb67001076c4134f82ad0dae5cef3704\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6148e980bdc4394e5ee766bac607fc607777a5c1c2dad4c18e243c006121235c\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d5547fcebf920ead6c1f429068078852ea910912bb1ac2942a5f05f8eea4e968\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:746e5cb3f7701cf0afcbea92f34ada0c4ea060fc2691a59ebd1ed8239a8e61b5\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:abd3eaae48639c784a21da52d7da93bd9ba38c938fc5941c9990958fd6b47f5d\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31e175b644c6627779ca6b9f48d9426760dd449788ad671dd0277cce8fb822e8\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69e134b501fdb83fc21e3e9e28b43d0f9ddf6643e2a87fc952d2ab48739121a9\\\",\\\"arohcpocpdev.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0fd69333c121efd68919e731cd3ea01e976e00415bda6e987bf1e949ec224eda\\\",\\\"arohcpocpdev.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:af3a4a739d80ee0b0d8a6c24b0dd9206d3e91907593da7cad54d881306b287ad\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3da5a449ace61cb7d002d8e7911c14905c74cf58a296527458d1af0f1abf0a72\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:464c7d227e1c80e413d87d99fdf42fc80d76b3bc8f9281a23b7d59b3b4ea2372\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6ea0f19030ffc52a290e6eeb812ef2426c48a2f1083b1e415e0667998cbb7bf2\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cf9f6b6fecd35f94d7cdb57845990602be060e4b548aac8ab6bc1c96e22f9bb2\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1c5b548d67e5c56e3297b2f8aa34428e544b5b5ac88bd8d0f465760375c501e6\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bc7e9750e4647b2ccd9a7c1c0f4cc35f640b36d8ab9d3fa789eb2764afce3d7a\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6b085cdad58db8529d99c22ec1ec79fe09ab00ab1c92ee38b29e7875dc0533c\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e2efd3418ad3517f5db50de37cda00d27bb568554e7cb618ade7c8eebfbc9185\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:24929612f2f0f2d4cd3ac163c04bbf4ac64cc47d3d0eef81db085d00c59058bd\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:987fe55afa2ee263467ecaf5c058664ad8e55126687f79d28ab5fa030ec17768\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cca646f56344329bd71967e6b044f43543f52f8c6a8986d725610ad92df26989\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d0d866dee27d9169a1e4711a35a4dfbc3ffe9655258a72188282f4c32eaa84a1\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2ade6dfb5406ef3a4264dea95d455298f54e8ef4d458d9ee3e94c33d48676a9b\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6ba2d747627cdfa2d7d36728dac5444865455d422df60f33943a45cef211badb\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b9f60077c475f2baad0dd51ffc9d469f3a9b54c474bb86a35f93cb9b8fdc734\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bccd3f544da031a055000bcf...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:214]: Timed out after 1800.003s.\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00090ba80\u003e: \n node pool upgrade verification failed: q0f3h5j5u6h0w3l-npupgrade-4-20-kdc5m-k9xcn (version 4.20 not in same minor as expected 4.21.5); nodes=[{\"name\":\"q0f3h5j5u6h0w3l-npupgrade-4-20-bgvp5-gwb6f\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.34.5-3.rhaos4.21.gita8af6ea.el9\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:70dcc3c7d3a572ecacc7c838fd17d07ff21531e50a7ed9227a17c27437151bf4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f33338fd94d039dc87a8f9c0fa00221c31bc493bedb70075f21a0fc912bfef04\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:185a95b130b6a27ad88fcf39803215b7bd998472b1e37225858435f4eb7c3c04\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ab5375f4de81dcf4e990e316124f3d9614ee588bc55342fe94b544602883afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:51e1b5dc9fccbee5b7df78d0e8bbe379f75529232c102c9ed5b1ae134c4b9b4c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:89c8b051a435031bdcc24715379e0003cb67001076c4134f82ad0dae5cef3704\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6148e980bdc4394e5ee766bac607fc607777a5c1c2dad4c18e243c006121235c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d5547fcebf920ead6c1f429068078852ea910912bb1ac2942a5f05f8eea4e968\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:746e5cb3f7701cf0afcbea92f34ada0c4ea060fc2691a59ebd1ed8239a8e61b5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:abd3eaae48639c784a21da52d7da93bd9ba38c938fc5941c9990958fd6b47f5d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31e175b644c6627779ca6b9f48d9426760dd449788ad671dd0277cce8fb822e8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69e134b501fdb83fc21e3e9e28b43d0f9ddf6643e2a87fc952d2ab48739121a9\",\"arohcpocpdev.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0fd69333c121efd68919e731cd3ea01e976e00415bda6e987bf1e949ec224eda\",\"arohcpocpdev.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:af3a4a739d80ee0b0d8a6c24b0dd9206d3e91907593da7cad54d881306b287ad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3da5a449ace61cb7d002d8e7911c14905c74cf58a296527458d1af0f1abf0a72\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:464c7d227e1c80e413d87d99fdf42fc80d76b3bc8f9281a23b7d59b3b4ea2372\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6ea0f19030ffc52a290e6eeb812ef2426c48a2f1083b1e415e0667998cbb7bf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cf9f6b6fecd35f94d7cdb57845990602be060e4b548aac8ab6bc1c96e22f9bb2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1c5b548d67e5c56e3297b2f8aa34428e544b5b5ac88bd8d0f465760375c501e6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bc7e9750e4647b2ccd9a7c1c0f4cc35f640b36d8ab9d3fa789eb2764afce3d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6b085cdad58db8529d99c22ec1ec79fe09ab00ab1c92ee38b29e7875dc0533c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e2efd3418ad3517f5db50de37cda00d27bb568554e7cb618ade7c8eebfbc9185\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:24929612f2f0f2d4cd3ac163c04bbf4ac64cc47d3d0eef81db085d00c59058bd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:987fe55afa2ee263467ecaf5c058664ad8e55126687f79d28ab5fa030ec17768\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cca646f56344329bd71967e6b044f43543f52f8c6a8986d725610ad92df26989\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d0d866dee27d9169a1e4711a35a4dfbc3ffe9655258a72188282f4c32eaa84a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2ade6dfb5406ef3a4264dea95d455298f54e8ef4d458d9ee3e94c33d48676a9b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6ba2d747627cdfa2d7d36728dac5444865455d422df60f33943a45cef211badb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b9f60077c475f2baad0dd51ffc9d469f3a9b54c474bb86a35f93cb9b8fdc734\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bccd3f544da031a055000bcf0a8fa1003dbe9cfa898bf2fb67c4ee3dcb4f4acc\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:17e68da37c74bdc95a4eb43b51ce671027807ee610a4ee5237324a8c0dc9d044\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b48b717cea29d8445ea717563e3794b381f6c1545b4458fb915c8baaa279893a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:12aec97ddeb527881acf5949bea2b227baa1dbfccf30679e8d57ff0b82e42707\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f88c407e2bdf86c7339ced2ae92bd716d5f74cee0999d41c5e3b26d47cfe2648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a26a47c35b590b15b04a9558c4fe3e34eef0fecfb4ab6e88274b7e9d70678a8d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c7f80064deb88df54bf40b4834fc5ffa9cd85fc18987fc82c0efdf579e228692\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2044d490f2d73d42bb6a114a2011c898034354e0fab1a872a723174b2c8327cc\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39a8fe4ceb03e1379951609b8d1ab33c40aa19f7fe24ed7b0f1621950c051cf6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:05ebc4de717d2d36e3d7daca025294e6de1181e3c67ebc38028fc0b0ea5bfafa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:23c2d9d4c3f769427c52b6b016633476fcf07a25523a0a3c6bcbf2a1b4840826\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:20a3e59566dd25058fff22e18569067fe6a12d8ac5ac62fa6508e0cee44d7897\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a405e40da35645d318b95c414f68536e3d3e2db22394ca9bf9738e666336e462\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:12448aad6e9050c2b2445035be818a490cef502d39b73dbd51cc0f5ea7d67d40\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:17a5d6b121e2bb5b8656a7a427824c2af9254ee0bd1f5cec30072b7c564eae7f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4c39dc633bf3560c2397281b74faf9d00500b0738dec6c1aad2cc8ce326be6fc\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ba77b4a636a254ef48caadc41666eb92b785317472708cf53d4a108cf7af9a78\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5b51a4d9d3b7ecad110fb6eb00f71e9e9b854067d79b220df397a58d9fa50eb0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9fe72e4ea5c3ef2600e9005c2ee869f9fbb68e31b569195e351a55ede1df703a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92a731e25801365be0ad0916bec5bf24b2035e443ad1414a1631d1e0473ddf77\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c91b7e9bdf73069bea2bc8ece90ce15c2a91a2bf99d9d970732b9afa233f3c6c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3f7181447dd3d28779323dac4561ca98d33755c19f468fb181d50c5338eca42a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:46427cb447a096a86618be4829484f964f10c220cd4d779e2c54a960c3045b7b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7e0a58aad44d5b09954f8261e3e428c8dfa076eb9c2b6ef46a91d27ac78ac17a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:83344c7d1374cbc2607b4fcb837f094c035b32daf24f8b3ccba2c8df409a6d3c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2f26472542da4f89a2ac34a93fd438d385419dda16811bdfa7e679e0115abe56\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e19f841ff1628bef075f117708faad18b1f525e6629542bb2db3031acac948fe\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3739c0c55f7f7ccc84309ae11eb23d3d9e5ad50a68e849a8a8dfda7f2f2bc137\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a20df9dd8819885aa431022c6f5e80b77b4431f268b719d71bec26f10521da0e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6263faa8ef7759ec21f26df08314c5d64e52d967b49961cb9404c54532c03365\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b37b3692dc74deb0b12b592173a09fb371a0a87210861f14239c70f247431294\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:72ae38682f25786aec18625cfed5d7831017d2f0f1304c3320c00ce0b7fb09c1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9540d66e9824b1ba081ad80bb6715e5ccf61b42c29c1ad961c1863efd81f48ca\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:14e983ca10e1033822e1c4be0a1e4a40762951a5d6e4a77bf642891f291d32ae\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:698e13ede97d3e2fbc3d69858b3f2d5985bbdfb3fb0f646452fe0edabbd5eb7f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:273930332d7b481bbca4521cf740507498a2a323a4010cd66e896806b299f85e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f942448d1f3d05bd5534d16433e0961509c35ea1eab6d51c0d2a8b7e16b0ad29\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1e6d59867cbd750672bc9ccfb9c69c3d198174cd43dd6d71a0da358b4b766473\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e99cd629414427a7792ccecda36d856dbe549682438f7b0dfb0ee2e7af536dd6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44ceafe5147680d58a5c2d3720932b972e6bc9639faadc19aa6b360af154e1a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5b3d41bea99611b73a21042f0f416dd3e8293c93086e18193c98b5fc1994734e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c59765325e6f286c067e555850b0d81f2de5e5ee8625ed83cdc2cea96f3a7ddb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cdd07910c008557d42ada95c4e41333704779166ffa8397b0adc73050a2df3ab\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1b58ba57c10e2ba03a0d8dba1f3f8faa866e1dca6170725b6909c2cc2b96efa6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f02f34239d2898fd346b775194a7e3f1a26b8aaeeaf3670b67cc5e4021742420\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e476b4dafff885727b0057259c9df67a68e196f8cd92c21db2e4efe84a5f7a25\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f6ee6b4bbee84922afa10d46091e1511788478855b20ce2ad7ac803852f06694\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ad039a130515504a258952cb18e67c3329da5182bb0623cc0891e6c15e2113a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c1f86751c05c53394f4fa5375c71b390f0908ea4a0a8dcc1be27d1aaad2b8130\",\"arohcpocpdev.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:4e62a2dcb8214491388d4c17f89fe699eb53cdb8039dcba6ab052edc28da3fbd\",\"arohcpocpdev.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:eea3f13f48a70c63b12c74b82caa6b1a8869130a43d5dc3c5ce7e518a4bdf4a6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cfcb0bd30c537e677ac6856c36c0c923b415f24d944c7aa1386489f511f6491d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f660d525da7c598367787fef3ac27294357524affe4f35c1f552e7f428ca7dab\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3e8d51b4b12d970bf561f67d0fcd9c0966a6d49866e6466daa341ee43d95d6e2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dadfaf3cceca2867f77c40b373b6fc80e34b4ba2cb1f05e7178a0cc79eac2d9d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:aa65d1c7fb8805b767fc4878faea280472f372010efadc66e5641a2e634a80c0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fcd01cbde0c58ad690f233a8f9ac9d080ca607fd7c24ad0de5afca564f098687\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:87de118dc29ca1b47d4768ec5ed4441bc1dada3fb2a78d53b1833249baa86dd4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b615eeec97d8429f11b1f8f9d05c7192c01ad293ed26c83b60bf6e31505f2c1f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64dbe77583c8b23dc6852fc482ea5b4081490734944ec8b765d5450b3fdb30ed\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9be8f12e3745100401dcf9f7d7e571260b11399f1dd98501208af756d08abd97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2e3661304917ce7f3be49974faacc60c24cbe2833d7b50c8c205c2f3f2467950\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d2afd0719d1a256d032048beeba2700cfe60da2904b2d9fe9c0aae3daf4318d5\",\"quay.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:1af59b7a29432314bde54e8977fa45fa92dc48885efbf0df601418ec0912f472\",\"quay.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:969bdfb00b7b9f7a99eb3892e34622055e6c98f72c74c64c9a60e1b4b881a246\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ad0e262327997e8cdc4074a2373e67274d9b8a943aa274a2cd8ce98ffe1db4a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d6c8c086bd9f73d1cc4a362ce6253914d4f4d8ce009b68aec42db9ff3a6aad36\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e5fa2595714628c16cf590efd28715bde790d2dc6b3a793b2a38dfb3ffb591a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5acacbb3eaaddf8d2f25e062427edd7f7636fc7a3b6a9fc84519df16fec0e5e0\"]},{\"name\":\"q0f3h5j5u6h0w3l-npupgrade-4-20-bgvp5-zf4gf\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.34.5-3.rhaos4.21.gita8af6ea.el9\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:70dcc3c7d3a572ecacc7c838fd17d07ff21531e50a7ed9227a17c27437151bf4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f33338fd94d039dc87a8f9c0fa00221c31bc493bedb70075f21a0fc912bfef04\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:185a95b130b6a27ad88fcf39803215b7bd998472b1e37225858435f4eb7c3c04\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ab5375f4de81dcf4e990e316124f3d9614ee588bc55342fe94b544602883afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:51e1b5dc9fccbee5b7df78d0e8bbe379f75529232c102c9ed5b1ae134c4b9b4c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:89c8b051a435031bdcc24715379e0003cb67001076c4134f82ad0dae5cef3704\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6148e980bdc4394e5ee766bac607fc607777a5c1c2dad4c18e243c006121235c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d5547fcebf920ead6c1f429068078852ea910912bb1ac2942a5f05f8eea4e968\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:746e5cb3f7701cf0afcbea92f34ada0c4ea060fc2691a59ebd1ed8239a8e61b5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:abd3eaae48639c784a21da52d7da93bd9ba38c938fc5941c9990958fd6b47f5d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31e175b644c6627779ca6b9f48d9426760dd449788ad671dd0277cce8fb822e8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69e134b501fdb83fc21e3e9e28b43d0f9ddf6643e2a87fc952d2ab48739121a9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3da5a449ace61cb7d002d8e7911c14905c74cf58a296527458d1af0f1abf0a72\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:464c7d227e1c80e413d87d99fdf42fc80d76b3bc8f9281a23b7d59b3b4ea2372\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6ea0f19030ffc52a290e6eeb812ef2426c48a2f1083b1e415e0667998cbb7bf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cf9f6b6fecd35f94d7cdb57845990602be060e4b548aac8ab6bc1c96e22f9bb2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1c5b548d67e5c56e3297b2f8aa34428e544b5b5ac88bd8d0f465760375c501e6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bc7e9750e4647b2ccd9a7c1c0f4cc35f640b36d8ab9d3fa789eb2764afce3d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6b085cdad58db8529d99c22ec1ec79fe09ab00ab1c92ee38b29e7875dc0533c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e2efd3418ad3517f5db50de37cda00d27bb568554e7cb618ade7c8eebfbc9185\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:24929612f2f0f2d4cd3ac163c04bbf4ac64cc47d3d0eef81db085d00c59058bd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:987fe55afa2ee263467ecaf5c058664ad8e55126687f79d28ab5fa030ec17768\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cca646f56344329bd71967e6b044f43543f52f8c6a8986d725610ad92df26989\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d0d866dee27d9169a1e4711a35a4dfbc3ffe9655258a72188282f4c32eaa84a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2ade6dfb5406ef3a4264dea95d455298f54e8ef4d458d9ee3e94c33d48676a9b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6ba2d747627cdfa2d7d36728dac5444865455d422df60f33943a45cef211badb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b9f60077c475f2baad0dd51ffc9d469f3a9b54c474bb86a35f93cb9b8fdc734\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bccd3f544da031a055000bcf0a8fa1003dbe9cfa898bf2fb67c4ee3dcb4f4acc\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:17e68da37c74bdc95a4eb43b51ce671027807ee610a4ee5237324a8c0dc9d044\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b48b717cea29d8445ea717563e3794b381f6c1545b4458fb915c8baaa279893a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:12aec97ddeb527881acf5949bea2b227baa1dbfccf30679e8d57ff0b82e42707\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f88c407e2bdf86c7339ced2ae92bd716d5f74cee0999d41c5e3b26d47cfe2648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a26a47c35b590b15b04a9558c4fe3e34eef0fecfb4ab6e88274b7e9d70678a8d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c7f80064deb88df54bf40b4834fc5ffa9cd85fc18987fc82c0efdf579e228692\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2044d490f2d73d42bb6a114a2011c898034354e0fab1a872a723174b2c8327cc\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:39a8fe4ceb03e1379951609b8d1ab33c40aa19f7fe24ed7b0f1621950c051cf6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:05ebc4de717d2d36e3d7daca025294e6de1181e3c67ebc38028fc0b0ea5bfafa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:23c2d9d4c3f769427c52b6b016633476fcf07a25523a0a3c6bcbf2a1b4840826\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5b51a4d9d3b7ecad110fb6eb00f71e9e9b854067d79b220df397a58d9fa50eb0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9fe72e4ea5c3ef2600e9005c2ee869f9fbb68e31b569195e351a55ede1df703a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:66640a3f7bd65090345552611701a73e74bc44897b9d29e43b11f73b96598ac0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:adcbb731ad586feb2c21dbc4993b26a564bbcb26efffb3f94f9da38e27002ea8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:92a731e25801365be0ad0916bec5bf24b2035e443ad1414a1631d1e0473ddf77\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c91b7e9bdf73069bea2bc8ece90ce15c2a91a2bf99d9d970732b9afa233f3c6c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3f7181447dd3d28779323dac4561ca98d33755c19f468fb181d50c5338eca42a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:46427cb447a096a86618be4829484f964f10c220cd4d779e2c54a960c3045b7b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7e0a58aad44d5b09954f8261e3e428c8dfa076eb9c2b6ef46a91d27ac78ac17a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:83344c7d1374cbc2607b4fcb837f094c035b32daf24f8b3ccba2c8df409a6d3c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3739c0c55f7f7ccc84309ae11eb23d3d9e5ad50a68e849a8a8dfda7f2f2bc137\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a20df9dd8819885aa431022c6f5e80b77b4431f268b719d71bec26f10521da0e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:72ae38682f25786aec18625cfed5d7831017d2f0f1304c3320c00ce0b7fb09c1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9540d66e9824b1ba081ad80bb6715e5ccf61b42c29c1ad961c1863efd81f48ca\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:14e983ca10e1033822e1c4be0a1e4a40762951a5d6e4a77bf642891f291d32ae\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:698e13ede97d3e2fbc3d69858b3f2d5985bbdfb3fb0f646452fe0edabbd5eb7f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:273930332d7b481bbca4521cf740507498a2a323a4010cd66e896806b299f85e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f942448d1f3d05bd5534d16433e0961509c35ea1eab6d51c0d2a8b7e16b0ad29\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1e6d59867cbd750672bc9ccfb9c69c3d198174cd43dd6d71a0da358b4b766473\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e99cd629414427a7792ccecda36d856dbe549682438f7b0dfb0ee2e7af536dd6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:44ceafe5147680d58a5c2d3720932b972e6bc9639faadc19aa6b360af154e1a3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5b3d41bea99611b73a21042f0f416dd3e8293c93086e18193c98b5fc1994734e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5bd409efe77c6a126418be2c098dde9431c1a290cec885a7e37c0c8ddc2337a8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c24f5d0b962f9b381df0f8d90d4c31616a95623fbca70de0a2da621aba6bd19f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2a07c1b2ef06bbfa3a491b84f7410e24de7cb487f49bbbb3bfd7cb3510c8429a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bd3da568f95c65b04a03d96d175d64efcabcb6fe9a454a1658bbfa243b16a0a7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c59765325e6f286c067e555850b0d81f2de5e5ee8625ed83cdc2cea96f3a7ddb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cdd07910c008557d42ada95c4e41333704779166ffa8397b0adc73050a2df3ab\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e476b4dafff885727b0057259c9df67a68e196f8cd92c21db2e4efe84a5f7a25\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f6ee6b4bbee84922afa10d46091e1511788478855b20ce2ad7ac803852f06694\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ad039a130515504a258952cb18e67c3329da5182bb0623cc0891e6c15e2113a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c1f86751c05c53394f4fa5375c71b390f0908ea4a0a8dcc1be27d1aaad2b8130\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cfcb0bd30c537e677ac6856c36c0c923b415f24d944c7aa1386489f511f6491d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f660d525da7c598367787fef3ac27294357524affe4f35c1f552e7f428ca7dab\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3e8d51b4b12d970bf561f67d0fcd9c0966a6d49866e6466daa341ee43d95d6e2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dadfaf3cceca2867f77c40b373b6fc80e34b4ba2cb1f05e7178a0cc79eac2d9d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5f852f6127d27e6fc071afe58d80024370467cb2476f4d9f077f8ebca8600918\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d6dd7872d95388063d6606823e4c4340f83a4f6983034f265a4d749f0418e77c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:aa65d1c7fb8805b767fc4878faea280472f372010efadc66e5641a2e634a80c0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fcd01cbde0c58ad690f233a8f9ac9d080ca607fd7c24ad0de5afca564f098687\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:87de118dc29ca1b47d4768ec5ed4441bc1dada3fb2a78d53b1833249baa86dd4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b615eeec97d8429f11b1f8f9d05c7192c01ad293ed26c83b60bf6e31505f2c1f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:64dbe77583c8b23dc6852fc482ea5b4081490734944ec8b765d5450b3fdb30ed\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9be8f12e3745100401dcf9f7d7e571260b11399f1dd98501208af756d08abd97\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2e3661304917ce7f3be49974faacc60c24cbe2833d7b50c8c205c2f3f2467950\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d2afd0719d1a256d032048beeba2700cfe60da2904b2d9fe9c0aae3daf4318d5\",\"quay.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:1af59b7a29432314bde54e8977fa45fa92dc48885efbf0df601418ec0912f472\",\"quay.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:969bdfb00b7b9f7a99eb3892e34622055e6c98f72c74c64c9a60e1b4b881a246\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ad0e262327997e8cdc4074a2373e67274d9b8a943aa274a2cd8ce98ffe1db4a0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d6c8c086bd9f73d1cc4a362ce6253914d4f4d8ce009b68aec42db9ff3a6aad36\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0e5fa2595714628c16cf590efd28715bde790d2dc6b3a793b2a38dfb3ffb591a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5acacbb3eaaddf8d2f25e062427edd7f7636fc7a3b6a9fc84519df16fec0e5e0\"]},{\"name\":\"q0f3h5j5u6h0w3l-npupgrade-4-20-kdc5m-k9xcn\",\"ready\":true,\"containerRuntimeVersion\":\"cri-o://1.33.9-3.rhaos4.20.gitb9ac835.el9\",\"releaseImages\":[\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:670004bffddb41c787b9f9a5c7d1615453d4acfa6452dedfa61ad2f305eea814\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:779115a9d5f9b0373950fdf99b8bb7bffb8a9143ec28c6e1558d1d3d8eec50e4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:70dcc3c7d3a572ecacc7c838fd17d07ff21531e50a7ed9227a17c27437151bf4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f33338fd94d039dc87a8f9c0fa00221c31bc493bedb70075f21a0fc912bfef04\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:185a95b130b6a27ad88fcf39803215b7bd998472b1e37225858435f4eb7c3c04\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ab5375f4de81dcf4e990e316124f3d9614ee588bc55342fe94b544602883afa\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3e262d5207d5e2077e4a14f4fbddc70d4d2651dbaaf955147013feb58a18cddd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8e08a29d3b206379957f1fb668058b4effe2f4542c2b4e45a3cfafa4b9a728e5\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:862d5befeba504d5058264cf65f16fc307eea4eb3914fbbbf40fbe61714b0958\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9aee9be2da68239a2a559e3ca1211eb8988a7f83a4ff82b9bd7c21a436be9c22\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:51e1b5dc9fccbee5b7df78d0e8bbe379f75529232c102c9ed5b1ae134c4b9b4c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:89c8b051a435031bdcc24715379e0003cb67001076c4134f82ad0dae5cef3704\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ecf24d093581fc39574dceec3d50efd4c3146cebf49bb7a5a450b3f8ce740551\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:fcdb0afaa38bf8deddfcda9abb50886c2a72c54695992d79c15a8413ac2d1796\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0cff1d43f1ec9e919965e5cedd2268233c0f7b27b07750bd6fe8a9bf10f65bf4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bf18d37657cff26bb88be6fbbdaf64ae8f1f090312233b0c50a904a35729c74c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7ea1ff613255c1ee67b34ec82b68ec6f5a4a6d283620a8e0ad142d4a31eb9211\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a45c25e5322761a9268bbaf5bdf3436d4623f42b90cd15942df0e5c22755e3ac\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:04c7a2673571e9069c934d86b4308c0f57b2715b45db500837b7708ae1f20ae7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:23016e42a746a81e3de1f5661cf2673efbe15ad0926b92d3d7c6fa9ac47f7215\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7dd9584d291967387f2a1ade991ad3bf8565bb59f49b013217951399e8273498\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f6ef619d31c4211120b23a7bf3118c9db7329c81b94709df86396d269d6b30e6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:03a7d45ceb838c746bdcdac1a5cc02d26273b319d9da4498bb3ee31c3e130b43\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:be6ed9469bb1acf2658aaf55b812e5334e308b4169b858b0bacf54ce6f4e2ca7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0c90ca6acbe2f4fdbd253ae22f5b3286247c6940321f7d2b9801aea21ef5269f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:26e4f5b039a340bc1ae080fb9d92799700b57ae5df45120768f516682d6cc54b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1adc4e7a9f123529e9df17eb1316a667d86ed2f185965c25ed6d32c44563188b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b963e4b54323ed252d951c924567590ccb4514d8e90b908c0be89f18919ae8cc\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31e175b644c6627779ca6b9f48d9426760dd449788ad671dd0277cce8fb822e8\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69e134b501fdb83fc21e3e9e28b43d0f9ddf6643e2a87fc952d2ab48739121a9\",\"arohcpocpdev.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0fd69333c121efd68919e731cd3ea01e976e00415bda6e987bf1e949ec224eda\",\"arohcpocpdev.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:af3a4a739d80ee0b0d8a6c24b0dd9206d3e91907593da7cad54d881306b287ad\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3da5a449ace61cb7d002d8e7911c14905c74cf58a296527458d1af0f1abf0a72\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:464c7d227e1c80e413d87d99fdf42fc80d76b3bc8f9281a23b7d59b3b4ea2372\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1652a6b3078719d56385dddda165a7e63168c645aa45ca408e7b12e682dcbdab\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7b0e43c637df9024af6222a689d54ba25e55f3a2a663202636b1b137b5a47a8c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6ea0f19030ffc52a290e6eeb812ef2426c48a2f1083b1e415e0667998cbb7bf2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cf9f6b6fecd35f94d7cdb57845990602be060e4b548aac8ab6bc1c96e22f9bb2\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:db40a3d4e50016c645fdb4a1f389dfcdbc9febc10c174816d681abe5c14153cd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f9b87244518245c63d1982182abe22775a2e207511cb328bc5c7b48e435b8da7\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1c5b548d67e5c56e3297b2f8aa34428e544b5b5ac88bd8d0f465760375c501e6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bc7e9750e4647b2ccd9a7c1c0f4cc35f640b36d8ab9d3fa789eb2764afce3d7a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:04ee35aa6f87e954d386228808017cf2938c5a96a052059c6767078d5f3ad8d6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:16dfb8b96a665be805bc1000677823aee946d93bfff0d3c567f2a24ea28a537a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6b085cdad58db8529d99c22ec1ec79fe09ab00ab1c92ee38b29e7875dc0533c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e2efd3418ad3517f5db50de37cda00d27bb568554e7cb618ade7c8eebfbc9185\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2a9f74fb0da7fb9f7d70c01953cb70fdda6a22bb9182bc5358f5fd6d957f4f6f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ef42cb417552fe1e38eedc30e92f55c8c50aae1aa1b0b7094ab56118efe0d96c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2a4f6a54cd5135c8121832236063419071fa5d00c10a051c197d3d01370e4d9c\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5dc767cd6267e006720af8cef541e23bc64d94ef9b1bf3a8b44b1fda5d9d8871\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:05376e7e2304d22dd866d8e25dd1e5ff8988b2232059fadfd12cb440af785635\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d6cf0a0701c4429a00483da429c3fa7d0f175d8e1602b2413860e8fc0c9f0e4f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9dee9b6866db30456c277ff1f7a114dd4ce25bff8aced70218623af9423ec3cf\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a3c906f4908c692697ae58891ef463ee62a23d4356bf00786947cc1dc02e4d53\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:78c23f5bccb6e82ade8428c8886ddc0291faec496590b2e6b5375ac6c0476dc3\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9ff94628410a702de0cb3e1a715917829b36c3def524dcc0850ac34a811f8f6f\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:876fb57ffb255ed75a452405bcd1d116788a34b19264886e6e087ff544c17083\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:dd6a017092d57ab3a6663f3822a0fa0e3bce4365859d9522dd173405276cd49a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ff360b712000d39cbb58158186ff0a18764278e6fa98ef9f21d113b978d9a79\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:77d8ddf2957025a39616328b5cf2b012de6cedce63fb79eebc5270d7b3401574\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:11a864539ae77a94afcf8d2e1abf9168081c5081a1bc06088f4be5206cdc02df\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2bcca887297fc1168fa43a6c1acb7077bf198258fe2e0ef9379e11c418919c15\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bbb5af28bbed69e8189825fdc6665f4e8dbcd0c0c83fa479943ea102fe9452d6\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c85dd52f4140690eef0d762834ab8b9af2faedbbd3bd9dd5616d4cd0bebd4740\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:124f1b391b0f376a5deffd05b29ef0896ca0912e1f67b3a2051ed6678c54a57e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ff8a8892f37210c582b3e526cbd2652b02ef9007d57be12b404434374dd0116a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:24929612f2f0f2d4cd3ac163c04bbf4ac64cc47d3d0eef81db085d00c59058bd\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:987fe55afa2ee263467ecaf5c058664ad8e55126687f79d28ab5fa030ec17768\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cca646f56344329bd71967e6b044f43543f52f8c6a8986d725610ad92df26989\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d0d866dee27d9169a1e4711a35a4dfbc3ffe9655258a72188282f4c32eaa84a1\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2ade6dfb5406ef3a4264dea95d455298f54e8ef4d458d9ee3e94c33d48676a9b\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6ba2d747627cdfa2d7d36728dac5444865455d422df60f33943a45cef211badb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:317131706dcce9c87eb4e1efb8446bdbf5e0726e120a8f6c655e2acd89d60e2d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:ec94607aa5fce47591b1ed1939e86a0e1c43cc8ff439f42662c9ed32e414595e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b9f60077c475f2baad0dd51ffc9d469f3a9b54c474bb86a35f93cb9b8fdc734\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bccd3f544da031a055000bcf0a8fa1003dbe9cfa898bf2fb67c4ee3dcb4f4acc\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2962c6c31d7078b34cf7ba0c9b0ea685dc539cea67b06bd38b6ac566298bb790\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:321163a14934467a86e358bbb3115fbe104ad814bcaf7423f259eb82150da703\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5efa227ddac82e56eddc03761e220af1f2d6a4b22ee00b2baf426523f26c0319\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d7376a666810d810549b4e270d9a2fd6bcfd8799370ff52476a4f799cd4e287d\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:602dc9f92b7a8e84e6eb998191263d186d1d10d378eeef0845435715bee812dc\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f7953a8e77c5bb0efaa670ba2188050740eba0e1f6979248832c05cb886fa5d0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:17e68da37c74bdc95a4eb43b51ce671027807ee610a4ee5237324a8c0dc9d044\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b48b717cea29d8445ea717563e3794b381f6c1545b4458fb915c8baaa279893a\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7f426776d3ef70b585018b8d813f5eedd70373536681c6240df224ebf9ff6329\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d2906e0869191da12130ed79a57221aae735ef026d0f909534c7cbfbca131128\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3d68a9523f4c5c22a48dfa151bbfde78b2f360697afe3b9d8bfb53e8c359bf95\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7983b528aa73931fdaa817858ecd416dea0ec5a6917bebf5bd11b4eb83c752c9\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8f13a1e673c882ee926e97270249a20d3c40797a8c7c5abb2fda9c98c24a2eb4\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d4067116d68c1aabb5ee21c4addce05f6849f104d7a7f87f5f1fcef7990a3aee\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2af652f68b17ed49c2565928e4c101880e949c4484c6ee958793a8e668e47bea\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e199029f436d7247195d0c5b22909cb874ac67bd87d84c597a83ef7bd0d87ab0\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:25d9148ea34149935323231c64952679727b6f7f235febc3be1df63e0bf18c5e\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c4fdcf25b69610f918eac662369ed457ee400b3d5483e8fcbecd44a97c220302\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:12aec97ddeb527881acf5949bea2b227baa1dbfccf30679e8d57ff0b82e42707\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f88c407e2bdf86c7339ced2ae92bd716d5f74cee0999d41c5e3b26d47cfe2648\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:41e0dcd2f91df9234249f168b3fbe6d55cfdb3ebd2e598d5edfa47f905c046cb\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9998ff90e081aac492498314bd02225cc3a8636d8c714ce8b618f4ee1de28c21\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:30a054cfe1b8330bd57d762921ca5cd281ad2fdd7994fd3093ffadb5d767a564\",\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:9b369532d212af90d1232bcf0a84bc2616106733f4eae8d4aaf69fad595e1d67\"]}]\n {\n s: \"node pool upgrade verification failed: q0f3h5j5u6h0w3l-npupgrade-4-20-kdc5m-k9xcn (version 4.20 not in same minor as expected 4.21.5); nodes=[{\\\"name\\\":\\\"q0f3h5j5u6h0w3l-npupgrade-4-20-bgvp5-gwb6f\\\",\\\"ready\\\":true,\\\"containerRuntimeVersion\\\":\\\"cri-o://1.34.5-3.rhaos4.21.gita8af6ea.el9\\\",\\\"releaseImages\\\":[\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:70dcc3c7d3a572ecacc7c838fd17d07ff21531e50a7ed9227a17c27437151bf4\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f33338fd94d039dc87a8f9c0fa00221c31bc493bedb70075f21a0fc912bfef04\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:185a95b130b6a27ad88fcf39803215b7bd998472b1e37225858435f4eb7c3c04\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1ab5375f4de81dcf4e990e316124f3d9614ee588bc55342fe94b544602883afa\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:51e1b5dc9fccbee5b7df78d0e8bbe379f75529232c102c9ed5b1ae134c4b9b4c\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:89c8b051a435031bdcc24715379e0003cb67001076c4134f82ad0dae5cef3704\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6148e980bdc4394e5ee766bac607fc607777a5c1c2dad4c18e243c006121235c\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d5547fcebf920ead6c1f429068078852ea910912bb1ac2942a5f05f8eea4e968\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:746e5cb3f7701cf0afcbea92f34ada0c4ea060fc2691a59ebd1ed8239a8e61b5\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:abd3eaae48639c784a21da52d7da93bd9ba38c938fc5941c9990958fd6b47f5d\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:31e175b644c6627779ca6b9f48d9426760dd449788ad671dd0277cce8fb822e8\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:69e134b501fdb83fc21e3e9e28b43d0f9ddf6643e2a87fc952d2ab48739121a9\\\",\\\"arohcpocpdev.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:0fd69333c121efd68919e731cd3ea01e976e00415bda6e987bf1e949ec224eda\\\",\\\"arohcpocpdev.azurecr.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:af3a4a739d80ee0b0d8a6c24b0dd9206d3e91907593da7cad54d881306b287ad\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3da5a449ace61cb7d002d8e7911c14905c74cf58a296527458d1af0f1abf0a72\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:464c7d227e1c80e413d87d99fdf42fc80d76b3bc8f9281a23b7d59b3b4ea2372\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6ea0f19030ffc52a290e6eeb812ef2426c48a2f1083b1e415e0667998cbb7bf2\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cf9f6b6fecd35f94d7cdb57845990602be060e4b548aac8ab6bc1c96e22f9bb2\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:1c5b548d67e5c56e3297b2f8aa34428e544b5b5ac88bd8d0f465760375c501e6\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bc7e9750e4647b2ccd9a7c1c0f4cc35f640b36d8ab9d3fa789eb2764afce3d7a\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:a6b085cdad58db8529d99c22ec1ec79fe09ab00ab1c92ee38b29e7875dc0533c\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:e2efd3418ad3517f5db50de37cda00d27bb568554e7cb618ade7c8eebfbc9185\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:24929612f2f0f2d4cd3ac163c04bbf4ac64cc47d3d0eef81db085d00c59058bd\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:987fe55afa2ee263467ecaf5c058664ad8e55126687f79d28ab5fa030ec17768\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cca646f56344329bd71967e6b044f43543f52f8c6a8986d725610ad92df26989\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d0d866dee27d9169a1e4711a35a4dfbc3ffe9655258a72188282f4c32eaa84a1\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:2ade6dfb5406ef3a4264dea95d455298f54e8ef4d458d9ee3e94c33d48676a9b\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:6ba2d747627cdfa2d7d36728dac5444865455d422df60f33943a45cef211badb\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:3b9f60077c475f2baad0dd51ffc9d469f3a9b54c474bb86a35f93cb9b8fdc734\\\",\\\"quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:bccd3f544da031a055000bcf...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output","failure_pattern_match":{"status":"clustered","failure_pattern_id":"04e2f680f05d99e2770a3ff3a1e2fa54da49fafe5c57d57c6669bd6935b5df3e","failure_pattern":"node pool upgrade verification failed: \u003cnode\u003e (version 4","search_query":"node pool upgrade verification failed: q0f3h5j5u6h0w3l-npupgrade-4-20-kdc5m-k9xcn (version 4"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["04e2f680f05d99e2770a3ff3a1e2fa54da49fafe5c57d57c6669bd6935b5df3e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037163580305969152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4645,"pr_state":"closed","pr_sha":"bee1a1cd93f4acac5347211a17395b3636cacde3","final_merged_sha":"c66c503c1b9d5df065741762f0079c31491b75e8","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-26T13:43:40Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"96866812f054999f9c01f2ec178dbdebfa4190d20619ca07e12e326ce70b70bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037163580305969152","occurred_at":"2026-03-26T13:43:40Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037157400506535936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4510,"pr_state":"closed","pr_sha":"a3c49a02699147584f90d5a78315b81d0226ff8a","final_merged_sha":"a3c49a02699147584f90d5a78315b81d0226ff8a","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-26T13:18:20Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037154744014737408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4485,"pr_state":"closed","pr_sha":"56bf314caa75ae5340defeb47d3ce32c4afb8aea","final_merged_sha":"56bf314caa75ae5340defeb47d3ce32c4afb8aea","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-26T13:06:30Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037152710083481600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4485,"pr_state":"closed","pr_sha":"56bf314caa75ae5340defeb47d3ce32c4afb8aea","final_merged_sha":"56bf314caa75ae5340defeb47d3ce32c4afb8aea","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-26T13:00:06Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"05e52348bd0b052d05daf597346c1f37d21127d7c0bd1d8ed4417e1eeb12059d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037152710083481600","occurred_at":"2026-03-26T13:00:06Z","failed_at":"e2e","signature_id":"27bd9bc515ae2d9bf1479ed9080230ef2a28364c15e486ecaec548d941f1046f","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001444f90\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2nbkt4, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-2nbkt4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2nbkt4, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2nbkt4, error: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-2nbkt4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2nbkt4, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009456e0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2nbkt4\",\n },\n \u003c*fmt.wrapErrors | 0xc001444f60\u003e{\n msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-2nbkt4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2nbkt4, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009456e0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2nbkt4\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001444f90\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2nbkt4, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-2nbkt4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2nbkt4, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2nbkt4, error: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-2nbkt4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2nbkt4, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009456e0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2nbkt4\",\n },\n \u003c*fmt.wrapErrors | 0xc001444f60\u003e{\n msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-2nbkt4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2nbkt4, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009456e0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-2nbkt4\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-48kgkn\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-48kgkn, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037151878831149056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4485,"pr_state":"closed","pr_sha":"56bf314caa75ae5340defeb47d3ce32c4afb8aea","final_merged_sha":"56bf314caa75ae5340defeb47d3ce32c4afb8aea","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-26T12:57:02Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4648/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037151306191212544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4648,"pr_state":"closed","pr_sha":"7c2f4e3fc239459ff7b1457bd14bb6e10e39b84a","final_merged_sha":"7c2f4e3fc239459ff7b1457bd14bb6e10e39b84a","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-26T12:54:54Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037147773681274880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4577,"pr_state":"closed","pr_sha":"9f3a000f1e01e0d9b552d1a9719e0d773e5473f0","final_merged_sha":"c168b54db271820581b99fe0ee96afc5c55827ce","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-26T12:40:52Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4647/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037147562540011520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4647,"pr_state":"closed","pr_sha":"cc6cdef062c74786d4f8c17830a6093b07d969fb","final_merged_sha":"cc6cdef062c74786d4f8c17830a6093b07d969fb","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-26T12:39:40Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037136162245578752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4473,"pr_state":"closed","pr_sha":"88bf2d442776b74296c7a9d5f76c506798f97079","final_merged_sha":"8b7576c81d01104dc238546ec2678952bf7afd61","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-26T11:54:43Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4c699d07a553d7ddc858fd238a29f6b964edd3c80c83a8571b4ec6ba3bc9f08e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037136162245578752","occurred_at":"2026-03-26T11:54:43Z","failed_at":"e2e","signature_id":"22a294f1bab9dae4ab18e3b62ae36cf2c7ae25439f1dc70c7a2e5c04d60cea3c","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:107]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011d91a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-48kgkn\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-48kgkn, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-48kgkn\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-48kgkn, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001196aa0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-48kgkn\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:107]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011d91a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-48kgkn\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-48kgkn, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-48kgkn\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-48kgkn, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001196aa0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-48kgkn\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-48kgkn\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-48kgkn, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4510/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037124973113380864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4510,"pr_state":"closed","pr_sha":"a3c49a02699147584f90d5a78315b81d0226ff8a","final_merged_sha":"a3c49a02699147584f90d5a78315b81d0226ff8a","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-26T11:10:14Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037122626249297920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4505,"pr_state":"closed","pr_sha":"ec8c17ac019d36b2be42061c5ce822f44607110b","final_merged_sha":"ec8c17ac019d36b2be42061c5ce822f44607110b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-26T11:00:56Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037121791037542400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4615,"pr_state":"closed","pr_sha":"b13100f96d3d60f7cc0590d31b88927416ce602f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-26T10:57:34Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4645/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037117848253370368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4645,"pr_state":"closed","pr_sha":"92e0c793269c098cabb62ae99ac5e260ea7ef1ef","final_merged_sha":"c66c503c1b9d5df065741762f0079c31491b75e8","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-26T10:41:56Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4485/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037110123272343552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4485,"pr_state":"closed","pr_sha":"56bf314caa75ae5340defeb47d3ce32c4afb8aea","final_merged_sha":"56bf314caa75ae5340defeb47d3ce32c4afb8aea","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-26T10:11:15Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4605/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037109963163176960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4605,"pr_state":"closed","pr_sha":"953df9ed36ff08f01a6be3767a1722b20faf615f","final_merged_sha":"953df9ed36ff08f01a6be3767a1722b20faf615f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-26T10:10:37Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037109296818294784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4505,"pr_state":"closed","pr_sha":"ec8c17ac019d36b2be42061c5ce822f44607110b","final_merged_sha":"ec8c17ac019d36b2be42061c5ce822f44607110b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-26T10:07:58Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"60283fc03a349a3ef660dadfecf09cfea1b1868956c40e6ee1b8eac64a71f2cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037109296818294784","occurred_at":"2026-03-26T10:07:58Z","failed_at":"provision","signature_id":"eece42ca4ffd5d6d0d8435a4000c11490cedb578474fa707f6bcbc98f49c7e45","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-26T10:48:23.528Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-26T10:48:24.177Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:2d906c8b-39a3-4047-8d40-cb975605ba4a cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j8294784] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j8294784.hcpsvc.osadev.cloud] image:map[digest:sha256:50b5fb5bf5e6af13c266dff43d6597d4e81da69b3fcc69a8884876fab568ea81 registry:arohcpsvcdev.azurecr.io repository:ci-op-6wcrgjtb/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-6wcrgjtb/pipeline:pull workloadIdentityClientId:277dd18f-32d4-4cb3-9d0f-7a85c95829b6 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:3301eb93-f945-4afd-b88e-96b8cc9739ba workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-03-26T10:48:24.177Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-26T10:48:24.382Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy namespace=sessiongate\ntime=2026-03-26T10:48:24.693Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy dryRun=false\ntime=2026-03-26T10:48:24.694Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-26T10:48:24.820Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-26T10:48:32.557Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF\"\ntime=2026-03-26T10:48:32.557Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-26T10:48:32.557Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-26T10:48:34.078Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy release=sessiongate namespace=sessiongate status=failed description=\"Release \\\"sessiongate\\\" failed: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF\" values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:2d906c8b-39a3-4047-8d40-cb975605ba4a cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-prow-j8294784] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j8294784.hcpsvc.osadev.cloud] image:map[digest:sha256:50b5fb5bf5e6af13c266dff43d6597d4e81da69b3fcc69a8884876fab568ea81 registry:arohcpsvcdev.azurecr.io repository:ci-op-6wcrgjtb/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-6wcrgjtb/pipeline:pull workloadIdentityClientId:277dd18f-32d4-4cb3-9d0f-7a85c95829b6 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:3301eb93-f945-4afd-b88e-96b8cc9739ba workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-03-26T10:48:34.081Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/6RUQWvbTBS861csOsXgFf6cED5cckiTQqFQjOtTSg+r1VjeeLVPvLeycfCPL4rtRji249Lb6mlmtPM0TOIRFUOoYQtRd6ow0USTe1xR/gwbv7lQjCSyC2VfbUffTYU/IywRXidSG7sf934mSimV/gAvncW9tdSEmPZVKhBxFEoT8e5xy3nwjUTwhPwhQu/OujLBlKiwldwTW8ZnFwoXykOipRCZvAfr/Cji7zQ8TAHW8LCxvc95zd0SLnP/iNrTem/tH/DagqObOdueGTOGzMEtyCxEO4mOtAslQ2Svdd/EObF7Ma2pMXln169472l1yDh6mbMCFSI7K1o+NnVWJ1Ccn1721+l0PKHmiPTxsDUSqZrs8v+ImQuu/WgHLlmHlxkmPbd1Zl4aRmapOkzg6dicidzlWTu1McvjxvtOdOvG+w+DaRlxzLR0BfjBm3c/Voutj0cm+fUpWTQ5OCBCvrQlIMlGreZgqOgqSDRVrXLEFRDUVWEi2vFVOhwMb/XgWg9vp/8NRjf/j4Y3T2lPZZk6g7kePqW9XrJRz+SCWrhQ3LkQwJ3qoqDeGqtbVYcdlWxUzdS+fbtoX9VUbMGXifSVp7KvKoiYspUkLsAqX3fMFxD7OwAA//8kk60oYAUAAA==\"\ntime=2026-03-26T10:48:34.082Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-03-26T10:48:34.083Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d8ac2e6cca70c7690a0f75cd38b523f3cb17701626a2f823018ba74b9a5ecb16","failure_pattern":"Internal error occurred: failed calling webhook \"validation.istio.io\": failed to call webhook: Post \"\u003curl\u003e\": EOF","search_query":"error running Helm release deployment Step, failed to deploy helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-26.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d8ac2e6cca70c7690a0f75cd38b523f3cb17701626a2f823018ba74b9a5ecb16"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4644/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037107324887240704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4644,"pr_state":"closed","pr_sha":"1c28bed858240edd30b21d68d7fea9966e1d27a2","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-26T10:00:08Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037105039314522112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4640,"pr_state":"closed","pr_sha":"ad178876c025d8ec012a031b98745809a6b44cc5","final_merged_sha":"e23d7c6b5233dccc23b0d4b24f1d727a143ec398","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-26T09:51:03Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4643/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037098947691941888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4643,"pr_state":"closed","pr_sha":"5e719eb4068d2526c4be22f983634af5a795fbb4","final_merged_sha":"5e719eb4068d2526c4be22f983634af5a795fbb4","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-26T09:26:51Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037098279841304576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4577,"pr_state":"closed","pr_sha":"9f3a000f1e01e0d9b552d1a9719e0d773e5473f0","final_merged_sha":"c168b54db271820581b99fe0ee96afc5c55827ce","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-26T09:24:11Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"1cc7ce8bfeab34a985cdad9f5f980b45682a3efb6a20f3073ff06cc0fcb2b54f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037098279841304576","occurred_at":"2026-03-26T09:24:11Z","failed_at":"e2e","signature_id":"8f187be618679d90d5fccdc9881f5e61ddf0a1c7ab4e19f0eef4f18aa8599569","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:124]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000d048d0\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-vsdn4b, error: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-vsdn4b\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-vsdn4b, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-vsdn4b\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0002c7a30\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-vsdn4b\",\n },\n \u003c*fmt.wrapError | 0xc00145e300\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-vsdn4b\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:124]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000d048d0\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-vsdn4b, error: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-vsdn4b\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-vsdn4b, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-vsdn4b\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0002c7a30\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-vsdn4b\",\n },\n \u003c*fmt.wrapError | 0xc00145e300\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-vsdn4b\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-vsdn4b, error: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-vsdn4b\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a7a7ffded83bfaaf3a4d7ab12e2848936cf72d3412903207b8997c2d2b17af58"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4642/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037096483198603264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4642,"pr_state":"closed","pr_sha":"5fe90812206ca7600153413f3489d4a5fcd46da1","final_merged_sha":"5fe90812206ca7600153413f3489d4a5fcd46da1","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-26T09:17:03Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4446/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037093684951912448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4446,"pr_state":"closed","pr_sha":"a76de5c4ea55d96336b4376361c6f1a3cd01f6f0","final_merged_sha":"1d16d503d07580c2671790287caed928be96c542","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-26T09:05:56Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"f7211e87ae2df168cd4969ad1119baedf05f570539f84e787b11605f8e2a1e32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4446/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037093684951912448","occurred_at":"2026-03-26T09:05:56Z","failed_at":"provision","signature_id":"f332a18ecda630e2a184fe05289bdad353416b3d510dcea88d09198c9377f182","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","test_suite":"step graph","failure_text":"time=2026-03-26T09:48:18.147Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-03-26T09:48:18.853Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[armPermissionsManagerIdentity:map[certName:armHelperCert2 clientId:3331e670-0804-48e8-a086-6241671ddc93 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj1912448.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj1912448.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clusterScopedIdentitiesConfig:map[roleSetName:dev] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:e9330836-104c-4ff6-8896-5e80a6561853 databaseName:arohcpprow-rp-j1912448 databaseUrl:https://arohcpprow-rp-j1912448.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-icg9x6nq/pipeline@sha256:d7affa8c94e1be1641298f474a1a1409323fb682a03fba66b2735fdfe48a905a replicas:2 zoneCount:4] exitOnPanic:true fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] miMock:map[certName:msiMockCert2 clientId:e8723db7-9b9e-46a4-9f7d-64d75c3534f0 principalId:d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-icg9x6nq/pipeline:pull workloadIdentityClientId:bbe90c53-94c0-41a0-a862-22f9c8836d86 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:e9330836-104c-4ff6-8896-5e80a6561853 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-26T09:48:18.853Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-26T09:48:18.912Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-03-26T09:48:19.026Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-03-26T09:48:19.026Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-26T09:48:19.137Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-26T09:53:23.504Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-03-26T09:53:23.504Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-26T09:53:23.504Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-26T09:53:27.747Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Release \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[armPermissionsManagerIdentity:map[certName:armHelperCert2 clientId:3331e670-0804-48e8-a086-6241671ddc93 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[cloudEnvironmentName:AzurePublicCloud msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj1912448.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj1912448.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clusterScopedIdentitiesConfig:map[roleSetName:dev] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:e9330836-104c-4ff6-8896-5e80a6561853 databaseName:arohcpprow-rp-j1912448 databaseUrl:https://arohcpprow-rp-j1912448.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-icg9x6nq/pipeline@sha256:d7affa8c94e1be1641298f474a1a1409323fb682a03fba66b2735fdfe48a905a replicas:2 zoneCount:4] exitOnPanic:true fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] maestro:map[sourceEnvironmentIdentifier:arohcpprow] miMock:map[certName:msiMockCert2 clientId:e8723db7-9b9e-46a4-9f7d-64d75c3534f0 principalId:d6b62dfa-87f5-49b3-bbcb-4a687c4faa96 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-icg9x6nq/pipeline:pull workloadIdentityClientId:bbe90c53-94c0-41a0-a862-22f9c8836d86 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:e9330836-104c-4ff6-8896-5e80a6561853 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-03-26T09:53:27.748Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/4xTTW8aMRS876942hNI64hCGzVUOaRpT1Uq1PSUqgevPV0cvH6rZy+IiB9fLSRhhSDhOm9m3oc9mUciQeRWDCJdk9VJJ116DLh8hEk/XLDTmMSFqqAd9FPXeIWwRNgisdHmBR7+yYiI8lsO/1x1p5u8oLzUZoFglX5qBUrakFwNZbaUrq6F1dw0efGG9iT7F3t0sIe2EAUPkxyHE8yvLlgXqr7z+8JvaDyva4TUK6pn/RH+jZFZ6/2RXk3rvSr3+IHwHrJ0BnccXGI5r9mz5sYYbncDvss9YqxqJHEmHtumTXMW96S768zYO7Pesrzn1YvsjflmgHQWCMkZfXDhM9rfwwjSTHjpLOTW6xj7F427ldQCa7XUre8/UZ79/ZIt2hISkBC/dx82ZhtazSGg7hPGpOuGSqQVEGhgdUIHD/LxaHypRhM1vvw9upp+/Dz9cPWQD+nigk5xPk2m48lDPhxmG3pkF2jhgr12IUB6MeNA+3T1Y3WYp2xDjXBX3Q9aUMN2Rz7PpCDPVUE1YtRVZ8liIVSue8tbRPM/AAD//3SFvhEMBAAA\"\ntime=2026-03-26T09:53:27.751Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-03-26T09:53:27.751Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f70296c5b0bdd447fa07f45b50524b3847ac3efdd92659ec6c4198951f8a52ee","failure_pattern":"resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f70296c5b0bdd447fa07f45b50524b3847ac3efdd92659ec6c4198951f8a52ee"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4510/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037093197502484480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4510,"pr_state":"closed","pr_sha":"a3c49a02699147584f90d5a78315b81d0226ff8a","final_merged_sha":"a3c49a02699147584f90d5a78315b81d0226ff8a","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-26T09:04:00Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"30a4b05ad0b521c2ea3f19ce01aa9d540c8682a163615624163826c6fd74fa04","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4510/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037093197502484480","occurred_at":"2026-03-26T09:04:00Z","failed_at":"e2e","signature_id":"c91213434333657d41364d783be4189a40478db55edd5c6dcc28ea950190705c","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:159]: Unexpected error:\n \u003c*errors.joinError | 0xc000b2a570\u003e: \n VerifyAllAPIServicesAvailable failed: the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f26be0\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then\",\n err: \u003c*errors.errorString | 0xc0007ff6e0\u003e{\n s: \"the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then\",\n },\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:159]: Unexpected error:\n \u003c*errors.joinError | 0xc000b2a570\u003e: \n VerifyAllAPIServicesAvailable failed: the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000f26be0\u003e{\n msg: \"VerifyAllAPIServicesAvailable failed: the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then\",\n err: \u003c*errors.errorString | 0xc0007ff6e0\u003e{\n s: \"the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then\",\n },\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"53c8663bb9391640cf5c05f563d13119c0eb49450a0c9d118656d27d9b9f12e7","failure_pattern":"VerifyAllAPIServicesAvailable failed: the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then","search_query":"VerifyAllAPIServicesAvailable failed: the APIService verifier is not allowed to re-try after Jan 15th, 2026 - HyperShift should only post ready status once aggregated APIs are ready then"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["53c8663bb9391640cf5c05f563d13119c0eb49450a0c9d118656d27d9b9f12e7"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4641/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037089595610370048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4641,"pr_state":"closed","pr_sha":"119265856cafc9dafafbabdf9105a8d7adfe4e00","final_merged_sha":"6a659af9d2b69b8b76999874fc48a3353e46370b","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-26T08:49:41Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4505/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037079293539913728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4505,"pr_state":"closed","pr_sha":"ec8c17ac019d36b2be42061c5ce822f44607110b","final_merged_sha":"ec8c17ac019d36b2be42061c5ce822f44607110b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-26T08:08:45Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4626/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037071925422854144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4626,"pr_state":"closed","pr_sha":"559c5455944773e805c348006b96e861b1569938","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-26T07:39:28Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4600/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037071178023047168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4600,"pr_state":"closed","pr_sha":"7de281f726d6f89eece80a8272da813d67560817","final_merged_sha":"7de281f726d6f89eece80a8272da813d67560817","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-26T07:36:30Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037069830569332736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4473,"pr_state":"closed","pr_sha":"8e38f2d1c45d2ef41e46ae8b28373875a0d7efbb","final_merged_sha":"8b7576c81d01104dc238546ec2678952bf7afd61","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-26T07:30:40Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"db62642aaac920dd1be62533007256a689bd5031142e875641fe5c9489a7c19e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037069830569332736","occurred_at":"2026-03-26T07:30:40Z","failed_at":"e2e","signature_id":"21e98140106e23218a0542e6344489fde7eb19999756a9ff634ef3e82754a58d","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:107]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001287800\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-cndfg8\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-cndfg8, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-cndfg8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-cndfg8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00140f540\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-cndfg8\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:107]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001287800\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-cndfg8\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-cndfg8, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-cndfg8\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-cndfg8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00140f540\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-cndfg8\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-48kgkn\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-48kgkn, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4640/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037054931378114560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4640,"pr_state":"closed","pr_sha":"ad178876c025d8ec012a031b98745809a6b44cc5","final_merged_sha":"e23d7c6b5233dccc23b0d4b24f1d727a143ec398","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-26T06:31:56Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4578/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037040095655104512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4578,"pr_state":"closed","pr_sha":"c34ea97833fcae1fb858af67f28c649ed55be4fa","final_merged_sha":"35d72d570de220208509fd109c0b0be041d2858a","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-26T05:32:59Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037035581875163136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4631,"pr_state":"closed","pr_sha":"c0f5f629de09b424d7aa42374069e6b9eccd6772","final_merged_sha":"c0f5f629de09b424d7aa42374069e6b9eccd6772","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-26T05:15:03Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037017952812797952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4631,"pr_state":"closed","pr_sha":"c0f5f629de09b424d7aa42374069e6b9eccd6772","final_merged_sha":"c0f5f629de09b424d7aa42374069e6b9eccd6772","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-26T04:05:00Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"01913c1f4c61cb6c0593235a446c6043f08c9dc838ae9e93f87580f88ac09d4f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037017952812797952","occurred_at":"2026-03-26T04:05:00Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037014020166258688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4631,"pr_state":"closed","pr_sha":"c0f5f629de09b424d7aa42374069e6b9eccd6772","final_merged_sha":"c0f5f629de09b424d7aa42374069e6b9eccd6772","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-26T03:49:19Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"3de7a5a2b77f348926d49b6d8033ec17c007faf04742baa2199a9505d21c47db","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037014020166258688","occurred_at":"2026-03-26T03:49:19Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037011642356600832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4631,"pr_state":"closed","pr_sha":"c0f5f629de09b424d7aa42374069e6b9eccd6772","final_merged_sha":"c0f5f629de09b424d7aa42374069e6b9eccd6772","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-26T03:39:55Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"7a05846acf6a16da5bdc48e3b36a262a48eefe4e2670b9cc7539f70b151b4863","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037011642356600832","occurred_at":"2026-03-26T03:39:55Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037009959555698689","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4631,"pr_state":"closed","pr_sha":"c0f5f629de09b424d7aa42374069e6b9eccd6772","final_merged_sha":"c0f5f629de09b424d7aa42374069e6b9eccd6772","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-26T03:33:14Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b6e2cc93cd86ecb5703f7691fdb067f5da07e10040e64ea25621166a0e768c82","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037009959555698689","occurred_at":"2026-03-26T03:33:14Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037005267740135424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4631,"pr_state":"closed","pr_sha":"c0f5f629de09b424d7aa42374069e6b9eccd6772","final_merged_sha":"c0f5f629de09b424d7aa42374069e6b9eccd6772","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-26T03:14:36Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"93c74d3ce999938a8ef82f878755ac59175d5c4d0d23c3075aa201599015c54a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037005267740135424","occurred_at":"2026-03-26T03:14:36Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4638/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037004038062477312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4638,"pr_state":"closed","pr_sha":"5ae5330a401280969d4a40da6fa115a4457c9122","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-26T03:09:42Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037000523126673408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4631,"pr_state":"closed","pr_sha":"c0f5f629de09b424d7aa42374069e6b9eccd6772","final_merged_sha":"c0f5f629de09b424d7aa42374069e6b9eccd6772","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-26T02:55:44Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"0219574d091ac04de771a7e2fae538aa586248a6947f1f67013b15363582e956","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2037000523126673408","occurred_at":"2026-03-26T02:55:44Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036998887431999488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4631,"pr_state":"closed","pr_sha":"c0f5f629de09b424d7aa42374069e6b9eccd6772","final_merged_sha":"c0f5f629de09b424d7aa42374069e6b9eccd6772","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-26T02:49:14Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"1a22f6483bbfe7592552f572105a7d8aaee92d9af99dbf340a2a5a37d78e65b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036998887431999488","occurred_at":"2026-03-26T02:49:14Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036985729401229312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4631,"pr_state":"closed","pr_sha":"c0f5f629de09b424d7aa42374069e6b9eccd6772","final_merged_sha":"c0f5f629de09b424d7aa42374069e6b9eccd6772","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-26T01:56:57Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6123363d6503d354765beeefd8401500c0ab851d6c4f886d09f0fb529bacb578","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036985729401229312","occurred_at":"2026-03-26T01:56:57Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036979493964353536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4631,"pr_state":"closed","pr_sha":"c0f5f629de09b424d7aa42374069e6b9eccd6772","final_merged_sha":"c0f5f629de09b424d7aa42374069e6b9eccd6772","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-26T01:32:11Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e668520e231085bb2a14a6afd84c13c750e1b7459edfc5ea15c431049fe4d957","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036979493964353536","occurred_at":"2026-03-26T01:32:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036977333948125184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4473,"pr_state":"closed","pr_sha":"8e38f2d1c45d2ef41e46ae8b28373875a0d7efbb","final_merged_sha":"8b7576c81d01104dc238546ec2678952bf7afd61","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-26T01:23:33Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"fbfd4bfc175996549fe7b51a989db5120fb269bdad746861d308269e541fb87a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4473/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036977333948125184","occurred_at":"2026-03-26T01:23:33Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036971068446478336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4631,"pr_state":"closed","pr_sha":"c0f5f629de09b424d7aa42374069e6b9eccd6772","final_merged_sha":"c0f5f629de09b424d7aa42374069e6b9eccd6772","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-26T00:58:42Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e4b64abc1d927f730a6a3a07423afca0426b048fc0bde6b122d9cfb6af7d4e94","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4631/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036971068446478336","occurred_at":"2026-03-26T00:58:42Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4626/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036968998620368896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4626,"pr_state":"closed","pr_sha":"559c5455944773e805c348006b96e861b1569938","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-26T00:50:28Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a83825616ee35a7db4ea929a6625e6976477e524db036c6585d7a02789551791","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4626/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036968998620368896","occurred_at":"2026-03-26T00:50:28Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036968304203010048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4626,"pr_state":"closed","pr_sha":"559c5455944773e805c348006b96e861b1569938","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-26T00:47:43Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"170111ed1185a24f9c26355053a6b933fca4c6e988f7ee35d9f274d61c4d4913","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036968304203010048","occurred_at":"2026-03-26T00:47:43Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4605/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036966225208152064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4605,"pr_state":"closed","pr_sha":"953df9ed36ff08f01a6be3767a1722b20faf615f","final_merged_sha":"953df9ed36ff08f01a6be3767a1722b20faf615f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-26T00:39:27Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"bbbd08725cd93f27776272abf078170c628b08eebb22a6828e87905cf641bfe5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4605/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036966225208152064","occurred_at":"2026-03-26T00:39:27Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036965557307183104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4605,"pr_state":"closed","pr_sha":"953df9ed36ff08f01a6be3767a1722b20faf615f","final_merged_sha":"953df9ed36ff08f01a6be3767a1722b20faf615f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-26T00:36:48Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"0f8ad4f04688fcf56fb017f78f33541488ec285d4706100a20ec06d4aed54cd7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2036965557307183104","occurred_at":"2026-03-26T00:36:48Z","failed_at":"provision","signature_id":"c027c4d12f6a3d02006d0282f3d5b1020328ed77f7bf43d76c7a5e7566e656ac","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-03-26T01:04:22.906Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-03-26T01:04:24.455Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-03-26T01:04:25.942Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=b222a08f24481e15cc453d548a8685e1bf6587357f329bda2c50ffce3e76ca14\ntime=2026-03-26T01:16:28.345Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7183104-svc/providers/Microsoft.Resources/deployments/b222a08f24481e15cc453d548a8685e1bf6587357f329bda2c50ffce3e76ca14/operationStatuses/08584271198203946776\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7183104-svc/providers/Microsoft.Resources/deployments/profile-imku7h4w54arm\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7183104-svc/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j7183104-svc/resourceAssociations/nsp-j7183104-svc-ra-b7209b60-e826-5723-b8f6-038fa1dd39d5\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AnmPreconditionFailedException\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Error while processing request for association /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7183104-svc/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j7183104-svc/resourceAssociations/nsp-j7183104-svc-ra-b7209b60-e826-5723-b8f6-038fa1dd39d5, Error message: There is already an operation in progress which requires exclusive lock on this service arohcpprow-rp-j7183104. Please retry the operation after sometime.\\\\\\\\r\\\\\\\\nActivityId: addf7b91-7865-44f6-bc6c-2062a8ac9d18, Microsoft.Azure.Documents.Common/2.14.0\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0f53a8239a3c5bbc9779fb8310071a301a0db32a497288816e31d1e2b5794349","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AnmPreconditionFailedException; detail message Error while processing network security perimeter association.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0f53a8239a3c5bbc9779fb8310071a301a0db32a497288816e31d1e2b5794349"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}