time=2026-03-20T23:48:44.111Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description="Step cluster\n  Kind: ARM\n  Template: templates/svc-cluster.bicep\n  Parameters: configurations/svc-cluster.tmpl.bicepparam"
time=2026-03-20T23:48:45.904Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster
time=2026-03-20T23:48:47.479Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=ca137337b764e3c08374540c6857f76d29e70e48046358dee986e5aa349b5ed0
time=2026-03-21T00:01:20.845Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j0646016-svc/providers/Microsoft.Resources/deployments/ca137337b764e3c08374540c6857f76d29e70e48046358dee986e5aa349b5ed0/operationStatuses/08584275563588572205\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/admin-api-cert-6yq2qgn2eojyi\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deploymentScripts/newCert-admin-api-cert-prow-j0646016\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"DeploymentScriptOperationFailed\\\",\\r\\n                \\\"message\\\": \\\"The request timed out. The tracking activity id is '3883f919-dadd-4ac6-9670-ce69949b6d5b', the correlation id is 'd15f5c50-8037-4187-a76c-bbf8da08ebb2'.\\\"\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:
    <*fmt.wrapErrors | 0xc0013b25a0>: 
    failed waiting for cluster="cluster-ver-4-19" in resourcegroup="rg-cluster-back-version-tbqvdk" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tbqvdk, error: context deadline exceeded
    {
        msg: "failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-tbqvdk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tbqvdk, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc0013eb8c0>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tbqvdk",
            },
            <context.deadlineExceededError>{},
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:
    <*fmt.wrapErrors | 0xc0013b25a0>: 
    failed waiting for cluster="cluster-ver-4-19" in resourcegroup="rg-cluster-back-version-tbqvdk" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tbqvdk, error: context deadline exceeded
    {
        msg: "failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-tbqvdk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tbqvdk, error: context deadline exceeded",
        errs: [
            <*errors.errorString | 0xc0013eb8c0>{
                s: "timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-tbqvdk",
            },
            <context.deadlineExceededError>{},
        ],
    }
occurred

time=2026-03-20T20:18:41.321Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image description="Step mirror-velero-image\n  Kind: ImageMirror\n  From {<nil> velero.server.registry <nil>}:{<nil> velero.server.repository <nil>}@{<nil> velero.server.digest <nil>} to {<nil> acr.svc.name <nil>}\n"
time=2026-03-20T20:19:14.034Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image err="error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\nLogging into target ACR arohcpsvcdev.\nFailed to log in to ACR arohcpsvcdev: ERROR: Could not connect to the registry login server 'arohcpsvcdev.azurecr.io'. Please verify that the registry exists and the URL 'https://arohcpsvcdev.azurecr.io/v2/' is reachable from your environment.\nTry running 'az acr check-health -n arohcpsvcdev --yes' to diagnose this issue.\n exit status 1"

fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:119]: Unexpected error:
    <*fmt.wrapError | 0xc0015d4c20>: 
    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation "idms-e2e-hcp-cluster" in resourcegroup="idms-872h6l": PUT https://rp.j8259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-872h6l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster
    --------------------------------------------------------------------------------
    RESPONSE 400: 400 Bad Request
    ERROR CODE: InvalidRequestContent
    --------------------------------------------------------------------------------
    {
      "error": {
        "code": "InvalidRequestContent",
        "message": "Required value: field cannot be null",
        "target": "properties.etcd.dataEncryption.customerManaged.kms.visibility"
      }
    }
    --------------------------------------------------------------------------------
    
    {
        msg: "failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-872h6l\": PUT https://rp.j8259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-872h6l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: InvalidRequestContent\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"InvalidRequestContent\",\n    \"message\": \"Required value: field cannot be null\",\n    \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n  }\n}\n--------------------------------------------------------------------------------\n",
        err: <*fmt.wrapError | 0xc0015d4c00>{
            msg: "failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-872h6l\": PUT https://rp.j8259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-872h6l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: InvalidRequestContent\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"InvalidRequestContent\",\n    \"message\": \"Required value: field cannot be null\",\n    \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n  }\n}\n--------------------------------------------------------------------------------\n",
            err: <*exported.ResponseError | 0xc0015f5920>{
                ErrorCode: "InvalidRequestContent",
                StatusCode: 400,
                RawResponse: {
                    Status: "400 Bad Request",
                    StatusCode: 400,
                    Proto: "HTTP/2.0",
                    ProtoMajor: 2,
                    ProtoMinor: 0,
                    Header: {
                        "X-Ms-Request-Id": [
                            "c1b4f007-c6e2-4269-baf5-c0064e14ae5e",
                        ],
                        "Date": [
                            "Fri, 20 Mar 2026 19:01:43 GMT",
                        ],
                        "Content-Length": ["206"],
                        "X-Envoy-Upstream-Service-Time": ["39"],
                        "Server": ["istio-envoy"],
                        "Content-Type": ["application/json"],
                        "X-Ms-Error-Code": [
                            "InvalidRequestContent",
                        ],
                    },
                    Body: <*exported.nopClosingBytesReader | 0xc0015d4b40>{
                        s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 82, 101, 113, 117, 105, 114, 101, 100, 32, 118, 97, 108, 117, 101, 58, 32, 102, 105, 101, 108, 100, 32, 99, 97, 110, 110, 111, 116, 32, 98, 101, 32, 110, 117, 108, 108, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 112, 114, 111, 112, 101, 114, 116, 105, 101, 115, 46, 101, 116, 99, 100, 46, 100, 97, 116, 97, 69, 110, 99, 114, 121, 112, 116, 105, 111, 110, 46, 99, 117, 115, 116, 111, 109, 101, 114, 77, 97, 110, 97, 103, 101, 100, 46, 107, 109, 115, 46, 118, 105, 115, 105, 98, 105, 108, 105, 116, 121, 34, 10, 32, 32, 32, 32, 125, 10, 125],
                        i: 0,
                    },
                    ContentLength: 206,
                    TransferEncoding: nil,
                ...

Gomega truncated this representation as it exceeds 'format.MaxLength'.
Consider having the object provide a custom 'GomegaStringer' representation
or adjust the parameters in Gomega's 'format' package.

Learn more here: https://onsi.github.io/gomega/#adjusting-output

occurred
fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:119]: Unexpected error:
    <*fmt.wrapError | 0xc0015d4c20>: 
    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation "idms-e2e-hcp-cluster" in resourcegroup="idms-872h6l": PUT https://rp.j8259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-872h6l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster
    --------------------------------------------------------------------------------
    RESPONSE 400: 400 Bad Request
    ERROR CODE: InvalidRequestContent
    --------------------------------------------------------------------------------
    {
      "error": {
        "code": "InvalidRequestContent",
        "message": "Required value: field cannot be null",
        "target": "properties.etcd.dataEncryption.customerManaged.kms.visibility"
      }
    }
    --------------------------------------------------------------------------------
    
    {
        msg: "failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-872h6l\": PUT https://rp.j8259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-872h6l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: InvalidRequestContent\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"InvalidRequestContent\",\n    \"message\": \"Required value: field cannot be null\",\n    \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n  }\n}\n--------------------------------------------------------------------------------\n",
        err: <*fmt.wrapError | 0xc0015d4c00>{
            msg: "failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-872h6l\": PUT https://rp.j8259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-872h6l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: InvalidRequestContent\n--------------------------------------------------------------------------------\n{\n  \"error\": {\n    \"code\": \"InvalidRequestContent\",\n    \"message\": \"Required value: field cannot be null\",\n    \"target\": \"properties.etcd.dataEncryption.customerManaged.kms.visibility\"\n  }\n}\n--------------------------------------------------------------------------------\n",
            err: <*exported.ResponseError | 0xc0015f5920>{
                ErrorCode: "InvalidRequestContent",
                StatusCode: 400,
                RawResponse: {
                    Status: "400 Bad Request",
                    StatusCode: 400,
                    Proto: "HTTP/2.0",
                    ProtoMajor: 2,
                    ProtoMinor: 0,
                    Header: {
                        "X-Ms-Request-Id": [
                            "c1b4f007-c6e2-4269-baf5-c0064e14ae5e",
                        ],
                        "Date": [
                            "Fri, 20 Mar 2026 19:01:43 GMT",
                        ],
                        "Content-Length": ["206"],
                        "X-Envoy-Upstream-Service-Time": ["39"],
                        "Server": ["istio-envoy"],
                        "Content-Type": ["application/json"],
                        "X-Ms-Error-Code": [
                            "InvalidRequestContent",
                        ],
                    },
                    Body: <*exported.nopClosingBytesReader | 0xc0015d4b40>{
                        s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 118, 97, 108, 105, 100, 82, 101, 113, 117, 101, 115, 116, 67, 111, 110, 116, 101, 110, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 82, 101, 113, 117, 105, 114, 101, 100, 32, 118, 97, 108, 117, 101, 58, 32, 102, 105, 101, 108, 100, 32, 99, 97, 110, 110, 111, 116, 32, 98, 101, 32, 110, 117, 108, 108, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 112, 114, 111, 112, 101, 114, 116, 105, 101, 115, 46, 101, 116, 99, 100, 46, 100, 97, 116, 97, 69, 110, 99, 114, 121, 112, 116, 105, 111, 110, 46, 99, 117, 115, 116, 111, 109, 101, 114, 77, 97, 110, 97, 103, 101, 100, 46, 107, 109, 115, 46, 118, 105, 115, 105, 98, 105, 108, 105, 116, 121, 34, 10, 32, 32, 32, 32, 125, 10, 125],
                        i: 0,
                    },
                    ContentLength: 206,
                    TransferEncoding: nil,
                ...

Gomega truncated this representation as it exceeds 'format.MaxLength'.
Consider having the object provide a custom 'GomegaStringer' representation
or adjust the parameters in Gomega's 'format' package.

Learn more here: https://onsi.github.io/gomega/#adjusting-output

occurred

fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_minor_upgrade.go:183]: Timed out after 2700.002s.
Expected success, but got an error:
    <*errors.errorString | 0xc0009e0190>: 
    no nodes found in node pool "npminorupgrade"
    {
        s: "no nodes found in node pool \"npminorupgrade\"",
    }
fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_minor_upgrade.go:183]: Timed out after 2700.002s.
Expected success, but got an error:
    <*errors.errorString | 0xc0009e0190>: 
    no nodes found in node pool "npminorupgrade"
    {
        s: "no nodes found in node pool \"npminorupgrade\"",
    }

time=2026-03-20T16:55:00.908Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster description="Step cluster\n  Kind: ARM\n  Template: templates/mgmt-cluster.bicep\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam"
time=2026-03-20T16:55:03.112Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster
time=2026-03-20T16:55:04.440Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster deployment=fb4a36dd0ff494fa35a9974bc8b6e8e5f0d46d48f18da1e495a99fed8805b844
time=2026-03-20T16:58:35.372Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j0361344-mgmt-1/providers/Microsoft.Resources/deployments/fb4a36dd0ff494fa35a9974bc8b6e8e5f0d46d48f18da1e495a99fed8805b844/operationStatuses/08584275811817549292\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0361344-mgmt-1/providers/Microsoft.Resources/deployments/cluster-wtgam6mzv3hok\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0361344-mgmt-1/providers/Microsoft.Resources/deployments/prow-j0361344-mgmt-1-outbound-ip\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n                \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j0361344-mgmt-1/providers/Microsoft.Resources/deployments/prow-j0361344-mgmt-1-outbound-ip\\\",\\r\\n                \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n                \\\"details\\\": [\\r\\n                  {\\r\\n                    \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n                    \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n                  }\\r\\n                ]\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-03-20T16:54:10.821Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster description="Step cluster\n  Kind: ARM\n  Template: templates/mgmt-cluster.bicep\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam"
time=2026-03-20T16:54:11.879Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster
time=2026-03-20T16:54:13.398Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster deployment=0d7e6aa49867da5a1f1c056487b5b629b56beeebeef7c241f5ff48c4143e8581
time=2026-03-20T16:57:44.411Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9497216-mgmt-1/providers/Microsoft.Resources/deployments/0d7e6aa49867da5a1f1c056487b5b629b56beeebeef7c241f5ff48c4143e8581/operationStatuses/08584275812329836622\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9497216-mgmt-1/providers/Microsoft.Resources/deployments/cluster-nvvsssnj7yrsk\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9497216-mgmt-1/providers/Microsoft.ContainerService/managedClusters/prow-j9497216-mgmt-1\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n                \\\"message\\\": \\\"Azure has a fixed 4000 role assignments limit per subscription. Please identify and delete useless role assingments before proceeding the operation. See https://learn.microsoft.com/en-us/azure/role-based-access-control/troubleshoot-limits for more details. Create role assignment failed. Scope: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9497216-mgmt-1-aks1; role assignment name: c7ba168c-98d3-4cc7-8bc5-d68c81ff6899. No more role assignments can be created.\\\",\\r\\n                \\\"details\\\": [\\r\\n                  {\\r\\n                    \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n                    \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n                  }\\r\\n                ]\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-03-20T16:45:51.997Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster description="Step cluster\n  Kind: ARM\n  Template: templates/mgmt-cluster.bicep\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam"
time=2026-03-20T16:45:52.845Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster
time=2026-03-20T16:45:54.113Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster deployment=9d866b4507ed610ff4cd012d65ac8a88c2797ef7dec33752bfeb6b0d5ca19bcf
time=2026-03-20T16:55:56.289Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7762688-mgmt-1/providers/Microsoft.Resources/deployments/9d866b4507ed610ff4cd012d65ac8a88c2797ef7dec33752bfeb6b0d5ca19bcf/operationStatuses/08584275817320700767\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7762688-mgmt-1/providers/Microsoft.Resources/deployments/metrics-infra\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7762688-mgmt-1/providers/Microsoft.Resources/deployments/1bfc0c3d-0803-54a4-98d0-9dd681a84c55\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7762688-mgmt-1/providers/Microsoft.Resources/deployments/e87ba3a5-5c8b-591c-ae06-77192812ed2b\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7762688-mgmt-1/providers/Microsoft.Resources/deployments/e8f77804-fe51-5e05-99f0-c3857de6fcd9\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7762688-mgmt-1/providers/Microsoft.Resources/deployments/6417f1c9-492b-5383-89d9-d3a268821cc8\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7762688-mgmt-1/providers/Microsoft.Resources/deployments/359981df-cf25-5199-bb86-db1b6021d9b5\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7762688-mgmt-1/providers/Microsoft.Resources/deployments/85f0b68f-9165-53f4-bad1-ddf2f1eaa33c\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-03-20T16:55:44.508Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions description="Step svc-mgmt-permissions\n  Kind: ARM\n  Template: templates/svc-mgmt-permissions.bicep\n  Parameters: configurations/svc-mgmt-permissions.tmpl.bicepparam"
time=2026-03-20T16:55:45.057Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions
time=2026-03-20T16:55:46.166Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions deployment=01e82a9dbd393631d4cdc403e6d2a13d0a5954cb6aa84fb7f837e5e0a6b958ce
time=2026-03-20T16:56:16.361Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=svc-mgmt-permissions err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/01e82a9dbd393631d4cdc403e6d2a13d0a5954cb6aa84fb7f837e5e0a6b958ce/operationStatuses/08584275811398670498\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/cx-backend-kv-access\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-3sr6ewk4usq2w\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n                \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-3sr6ewk4usq2w\\\",\\r\\n                \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n                \\\"details\\\": [\\r\\n                  {\\r\\n                    \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n                    \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n                  },\\r\\n                  {\\r\\n                    \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n                    \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n                  }\\r\\n                ]\\r\\n              }\\r\\n            ]\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-kzzwmv6egrcdg\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n                \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-kzzwmv6egrcdg\\\",\\r\\n                \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n                \\\"details\\\": [\\r\\n                  {\\r\\n                    \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n                    \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n                  },\\r\\n                  {\\r\\n                    \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n                    \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n                  }\\r\\n                ]\\r\\n              }\\r\\n            ]\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/cx-access-kzzwmv6egrcdg\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n                \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/cx-access-kzzwmv6egrcdg\\\",\\r\\n                \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n                \\\"details\\\": [\\r\\n                  {\\r\\n                    \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n                    \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n                  },\\r\\n                  {\\r\\n                    \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n                    \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n                  }\\r\\n                ]\\r\\n              }\\r\\n            ]\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/cx-access-3sr6ewk4usq2w\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n                \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/cx-access-3sr6ewk4usq2w\\\",\\r\\n                \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n                \\\"details\\\": [\\r\\n                  {\\r\\n                    \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n                    \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n                  },\\r\\n                  {\\r\\n                    \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n                    \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n                  }\\r\\n                ]\\r\\n              }\\r\\n            ]\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/cx-access-airyajwecijgk\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n                \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/cx-access-airyajwecijgk\\\",\\r\\n                \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n                \\\"details\\\": [\\r\\n                  {\\r\\n                    \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n                    \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n                  },\\r\\n                  {\\r\\n                    \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n                    \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n                  }\\r\\n                ]\\r\\n              }\\r\\n            ]\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-airyajwecijgk\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n                \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-airyajwecijgk\\\",\\r\\n                \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n                \\\"details\\\": [\\r\\n                  {\\r\\n                    \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n                    \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n                  },\\r\\n                  {\\r\\n                    \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n                    \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n                  }\\r\\n                ]\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-refresher-msi-kv-access\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-tpqc7lbi73vw6\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n                \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-tpqc7lbi73vw6\\\",\\r\\n                \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n                \\\"details\\\": [\\r\\n                  {\\r\\n                    \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n                    \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n                  }\\r\\n                ]\\r\\n              }\\r\\n            ]\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-77z4miyuwcj5q\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n                \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-77z4miyuwcj5q\\\",\\r\\n                \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n                \\\"details\\\": [\\r\\n                  {\\r\\n                    \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n                    \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n                  }\\r\\n                ]\\r\\n              }\\r\\n            ]\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-rskzz5hm4jrzk\\\",\\r\\n            \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n            \\\"details\\\": [\\r\\n              {\\r\\n                \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n                \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j9695872-mgmt-1/providers/Microsoft.Resources/deployments/msi-access-rskzz5hm4jrzk\\\",\\r\\n                \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n                \\\"details\\\": [\\r\\n                  {\\r\\n                    \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n                    \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n                  }\\r\\n                ]\\r\\n              }\\r\\n            ]\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"BadRequest\",\n        \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n    \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"BadRequest\",\n        \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n    \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

time=2026-03-20T16:43:07.752Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description="Step cluster\n  Kind: ARM\n  Template: templates/svc-cluster.bicep\n  Parameters: configurations/svc-cluster.tmpl.bicepparam"
time=2026-03-20T16:43:10.492Z level=DEBUG+2 msg="Starting ARM deployment" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster
time=2026-03-20T16:43:12.010Z level=DEBUG+3 msg="Deployment started" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=15335b63c7222269b741262fad17b1eea805682b1397c6a9e0508ec3969cac63
time=2026-03-20T16:58:15.605Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err="failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j6026368-svc/providers/Microsoft.Resources/deployments/15335b63c7222269b741262fad17b1eea805682b1397c6a9e0508ec3969cac63/operationStatuses/08584275818942771261\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: DeploymentFailed\n--------------------------------------------------------------------------------\n{\n  \"status\": \"Failed\",\n  \"error\": {\n    \"code\": \"DeploymentFailed\",\n    \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n    \"details\": [\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/aks-svc-kv-access-admin-api-cert-prow-j6026368\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/aks-svc-kv-access-frontend-cert-prow-j6026368\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/aksSPCRead-sessiongate-cert-prow-j6026368\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      },\n      {\n        \"code\": \"Conflict\",\n        \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j6026368-svc/providers/Microsoft.Resources/deployments/metrics-infra\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n      }\n    ]\n  }\n}\n--------------------------------------------------------------------------------\n"

fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:120]: Unexpected error:
    <*fmt.wrapError | 0xc0014d29c0>: 
    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster="idms-e2e-hcp-cluster" in resourcegroup="idms-mxzr74" to finish creating: GET https://rp.j7768832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1cf7d87-0eb8-45b2-942c-277542c40fb0
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1cf7d87-0eb8-45b2-942c-277542c40fb0",
      "name": "a1cf7d87-0eb8-45b2-942c-277542c40fb0",
      "status": "Failed",
      "startTime": "2026-03-20T15:47:06.242699653Z",
      "endTime": "2026-03-20T15:47:22.298584679Z",
      "error": {
        "code": "InternalServerError",
        "message": "SAL validation failed with status code: 400"
      }
    }
    --------------------------------------------------------------------------------
    
    {
        msg: "failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mxzr74\" to finish creating: GET https://rp.j7768832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1cf7d87-0eb8-45b2-942c-277542c40fb0\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1cf7d87-0eb8-45b2-942c-277542c40fb0\",\n  \"name\": \"a1cf7d87-0eb8-45b2-942c-277542c40fb0\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-03-20T15:47:06.242699653Z\",\n  \"endTime\": \"2026-03-20T15:47:22.298584679Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"SAL validation failed with status code: 400\"\n  }\n}\n--------------------------------------------------------------------------------\n",
        err: <*fmt.wrapError | 0xc0014d29a0>{
            msg: "failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mxzr74\" to finish creating: GET https://rp.j7768832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1cf7d87-0eb8-45b2-942c-277542c40fb0\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1cf7d87-0eb8-45b2-942c-277542c40fb0\",\n  \"name\": \"a1cf7d87-0eb8-45b2-942c-277542c40fb0\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-03-20T15:47:06.242699653Z\",\n  \"endTime\": \"2026-03-20T15:47:22.298584679Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"SAL validation failed with status code: 400\"\n  }\n}\n--------------------------------------------------------------------------------\n",
            err: <*exported.ResponseError | 0xc0014f8c60>{
                ErrorCode: "InternalServerError",
                StatusCode: 200,
                RawResponse: {
                    Status: "200 OK",
                    StatusCode: 200,
                    Proto: "HTTP/2.0",
                    ProtoMajor: 2,
                    ProtoMinor: 0,
                    Header: {
                        "Content-Type": ["application/json"],
                        "X-Ms-Request-Id": [
                            "2de212f1-7a31-4e59-8625-db2c8547d4f7",
                        ],
                        "Date": [
                            "Fri, 20 Mar 2026 15:47:26 GMT",
                        ],
                        "Content-Length": ["481"],
                        "X-Envoy-Upstream-Service-Time": ["10"],
                        "Server": ["istio-envoy"],
                    },
                    Body: <*exported.nopClosingBytesReader | 0xc0014d28a0>{
                        s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101...

Gomega truncated this representation as it exceeds 'format.MaxLength'.
Consider having the object provide a custom 'GomegaStringer' representation
or adjust the parameters in Gomega's 'format' package.

Learn more here: https://onsi.github.io/gomega/#adjusting-output

occurred
fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:120]: Unexpected error:
    <*fmt.wrapError | 0xc0014d29c0>: 
    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster="idms-e2e-hcp-cluster" in resourcegroup="idms-mxzr74" to finish creating: GET https://rp.j7768832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1cf7d87-0eb8-45b2-942c-277542c40fb0
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1cf7d87-0eb8-45b2-942c-277542c40fb0",
      "name": "a1cf7d87-0eb8-45b2-942c-277542c40fb0",
      "status": "Failed",
      "startTime": "2026-03-20T15:47:06.242699653Z",
      "endTime": "2026-03-20T15:47:22.298584679Z",
      "error": {
        "code": "InternalServerError",
        "message": "SAL validation failed with status code: 400"
      }
    }
    --------------------------------------------------------------------------------
    
    {
        msg: "failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mxzr74\" to finish creating: GET https://rp.j7768832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1cf7d87-0eb8-45b2-942c-277542c40fb0\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1cf7d87-0eb8-45b2-942c-277542c40fb0\",\n  \"name\": \"a1cf7d87-0eb8-45b2-942c-277542c40fb0\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-03-20T15:47:06.242699653Z\",\n  \"endTime\": \"2026-03-20T15:47:22.298584679Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"SAL validation failed with status code: 400\"\n  }\n}\n--------------------------------------------------------------------------------\n",
        err: <*fmt.wrapError | 0xc0014d29a0>{
            msg: "failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-mxzr74\" to finish creating: GET https://rp.j7768832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1cf7d87-0eb8-45b2-942c-277542c40fb0\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1cf7d87-0eb8-45b2-942c-277542c40fb0\",\n  \"name\": \"a1cf7d87-0eb8-45b2-942c-277542c40fb0\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-03-20T15:47:06.242699653Z\",\n  \"endTime\": \"2026-03-20T15:47:22.298584679Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"SAL validation failed with status code: 400\"\n  }\n}\n--------------------------------------------------------------------------------\n",
            err: <*exported.ResponseError | 0xc0014f8c60>{
                ErrorCode: "InternalServerError",
                StatusCode: 200,
                RawResponse: {
                    Status: "200 OK",
                    StatusCode: 200,
                    Proto: "HTTP/2.0",
                    ProtoMajor: 2,
                    ProtoMinor: 0,
                    Header: {
                        "Content-Type": ["application/json"],
                        "X-Ms-Request-Id": [
                            "2de212f1-7a31-4e59-8625-db2c8547d4f7",
                        ],
                        "Date": [
                            "Fri, 20 Mar 2026 15:47:26 GMT",
                        ],
                        "Content-Length": ["481"],
                        "X-Envoy-Upstream-Service-Time": ["10"],
                        "Server": ["istio-envoy"],
                    },
                    Body: <*exported.nopClosingBytesReader | 0xc0014d28a0>{
                        s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 116, 117, 115, 101...

Gomega truncated this representation as it exceeds 'format.MaxLength'.
Consider having the object provide a custom 'GomegaStringer' representation
or adjust the parameters in Gomega's 'format' package.

Learn more here: https://onsi.github.io/gomega/#adjusting-output

occurred

fail [github.com/Azure/ARO-HCP/test/e2e/keyvault_private.go:105]: Unexpected error:
    <*fmt.wrapError | 0xc000b08f40>: 
    failed to create HCP cluster private-key-vault: failed waiting for cluster="private-key-vault" in resourcegroup="private-key-vault-mbcpfw" to finish creating: GET https://rp.j7768832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03fab8fe-4421-4f9f-886d-eea23eadbe94
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03fab8fe-4421-4f9f-886d-eea23eadbe94",
      "name": "03fab8fe-4421-4f9f-886d-eea23eadbe94",
      "status": "Failed",
      "startTime": "2026-03-20T15:46:52.395992159Z",
      "endTime": "2026-03-20T15:47:12.443442695Z",
      "error": {
        "code": "InternalServerError",
        "message": "SAL validation failed with status code: 400"
      }
    }
    --------------------------------------------------------------------------------
    
    {
        msg: "failed to create HCP cluster private-key-vault: failed waiting for cluster=\"private-key-vault\" in resourcegroup=\"private-key-vault-mbcpfw\" to finish creating: GET https://rp.j7768832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03fab8fe-4421-4f9f-886d-eea23eadbe94\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03fab8fe-4421-4f9f-886d-eea23eadbe94\",\n  \"name\": \"03fab8fe-4421-4f9f-886d-eea23eadbe94\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-03-20T15:46:52.395992159Z\",\n  \"endTime\": \"2026-03-20T15:47:12.443442695Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"SAL validation failed with status code: 400\"\n  }\n}\n--------------------------------------------------------------------------------\n",
        err: <*fmt.wrapError | 0xc000b08f20>{
            msg: "failed waiting for cluster=\"private-key-vault\" in resourcegroup=\"private-key-vault-mbcpfw\" to finish creating: GET https://rp.j7768832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03fab8fe-4421-4f9f-886d-eea23eadbe94\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03fab8fe-4421-4f9f-886d-eea23eadbe94\",\n  \"name\": \"03fab8fe-4421-4f9f-886d-eea23eadbe94\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-03-20T15:46:52.395992159Z\",\n  \"endTime\": \"2026-03-20T15:47:12.443442695Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"SAL validation failed with status code: 400\"\n  }\n}\n--------------------------------------------------------------------------------\n",
            err: <*exported.ResponseError | 0xc00121b620>{
                ErrorCode: "InternalServerError",
                StatusCode: 200,
                RawResponse: {
                    Status: "200 OK",
                    StatusCode: 200,
                    Proto: "HTTP/2.0",
                    ProtoMajor: 2,
                    ProtoMinor: 0,
                    Header: {
                        "Content-Type": ["application/json"],
                        "X-Ms-Request-Id": [
                            "efbd221b-3753-44b9-920e-1119860e1071",
                        ],
                        "Date": [
                            "Fri, 20 Mar 2026 15:47:12 GMT",
                        ],
                        "Content-Length": ["481"],
                        "X-Envoy-Upstream-Service-Time": ["9"],
                        "Server": ["istio-envoy"],
                    },
                    Body: <*exported.nopClosingBytesReader | 0xc000b08d00>{
                        s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 11...

Gomega truncated this representation as it exceeds 'format.MaxLength'.
Consider having the object provide a custom 'GomegaStringer' representation
or adjust the parameters in Gomega's 'format' package.

Learn more here: https://onsi.github.io/gomega/#adjusting-output

occurred
fail [github.com/Azure/ARO-HCP/test/e2e/keyvault_private.go:105]: Unexpected error:
    <*fmt.wrapError | 0xc000b08f40>: 
    failed to create HCP cluster private-key-vault: failed waiting for cluster="private-key-vault" in resourcegroup="private-key-vault-mbcpfw" to finish creating: GET https://rp.j7768832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03fab8fe-4421-4f9f-886d-eea23eadbe94
    --------------------------------------------------------------------------------
    RESPONSE 200: 200 OK
    ERROR CODE: InternalServerError
    --------------------------------------------------------------------------------
    {
      "id": "/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03fab8fe-4421-4f9f-886d-eea23eadbe94",
      "name": "03fab8fe-4421-4f9f-886d-eea23eadbe94",
      "status": "Failed",
      "startTime": "2026-03-20T15:46:52.395992159Z",
      "endTime": "2026-03-20T15:47:12.443442695Z",
      "error": {
        "code": "InternalServerError",
        "message": "SAL validation failed with status code: 400"
      }
    }
    --------------------------------------------------------------------------------
    
    {
        msg: "failed to create HCP cluster private-key-vault: failed waiting for cluster=\"private-key-vault\" in resourcegroup=\"private-key-vault-mbcpfw\" to finish creating: GET https://rp.j7768832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03fab8fe-4421-4f9f-886d-eea23eadbe94\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03fab8fe-4421-4f9f-886d-eea23eadbe94\",\n  \"name\": \"03fab8fe-4421-4f9f-886d-eea23eadbe94\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-03-20T15:46:52.395992159Z\",\n  \"endTime\": \"2026-03-20T15:47:12.443442695Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"SAL validation failed with status code: 400\"\n  }\n}\n--------------------------------------------------------------------------------\n",
        err: <*fmt.wrapError | 0xc000b08f20>{
            msg: "failed waiting for cluster=\"private-key-vault\" in resourcegroup=\"private-key-vault-mbcpfw\" to finish creating: GET https://rp.j7768832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03fab8fe-4421-4f9f-886d-eea23eadbe94\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03fab8fe-4421-4f9f-886d-eea23eadbe94\",\n  \"name\": \"03fab8fe-4421-4f9f-886d-eea23eadbe94\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-03-20T15:46:52.395992159Z\",\n  \"endTime\": \"2026-03-20T15:47:12.443442695Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"SAL validation failed with status code: 400\"\n  }\n}\n--------------------------------------------------------------------------------\n",
            err: <*exported.ResponseError | 0xc00121b620>{
                ErrorCode: "InternalServerError",
                StatusCode: 200,
                RawResponse: {
                    Status: "200 OK",
                    StatusCode: 200,
                    Proto: "HTTP/2.0",
                    ProtoMajor: 2,
                    ProtoMinor: 0,
                    Header: {
                        "Content-Type": ["application/json"],
                        "X-Ms-Request-Id": [
                            "efbd221b-3753-44b9-920e-1119860e1071",
                        ],
                        "Date": [
                            "Fri, 20 Mar 2026 15:47:12 GMT",
                        ],
                        "Content-Length": ["481"],
                        "X-Envoy-Upstream-Service-Time": ["9"],
                        "Server": ["istio-envoy"],
                    },
                    Body: <*exported.nopClosingBytesReader | 0xc000b08d00>{
                        s: [123, 10, 32, 32, 32, 32, 34, 105, 100, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 55, 52, 101, 98, 100, 52, 54, 45, 56, 97, 100, 51, 45, 52, 49, 101, 51, 45, 97, 102, 101, 102, 45, 55, 101, 102, 50, 53, 102, 100, 53, 99, 51, 55, 49, 47, 112, 114, 111, 118, 105, 100, 101, 114, 115, 47, 77, 105, 99, 114, 111, 115, 111, 102, 116, 46, 82, 101, 100, 72, 97, 116, 79, 112, 101, 110, 83, 104, 105, 102, 116, 47, 108, 111, 99, 97, 116, 105, 111, 110, 115, 47, 119, 101, 115, 116, 117, 115, 51, 47, 104, 99, 112, 79, 112, 101, 114, 97, 116, 105, 111, 110, 83, 116, 97, 11...

Gomega truncated this representation as it exceeds 'format.MaxLength'.
Consider having the object provide a custom 'GomegaStringer' representation
or adjust the parameters in Gomega's 'format' package.

Learn more here: https://onsi.github.io/gomega/#adjusting-output

occurred

time=2026-03-20T15:10:05.652Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=global step=mirror-image description="Step mirror-image\n  Kind: ImageMirror\n  From {<nil> adminApi.image.registry <nil>}:{<nil> adminApi.image.repository <nil>}@{<nil> adminApi.image.digest <nil>} to {<nil> acr.svc.name <nil>}\n"
time=2026-03-20T15:10:47.848Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=global step=mirror-image err="error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\nSetting up registry authentication for CI source registry.\ninfo: Using registry public hostname registry.build02.ci.openshift.org\nSaved credentials for registry.build02.ci.openshift.org into /tmp/tmp.kN6XNGprsf/containers/auth.json\nLogging into target ACR arohcpsvcdev.\nLogin Succeeded\nMirroring image registry.build02.ci.openshift.org/ci-op-c5iz4fkw/pipeline@sha256:60efa6e90e760e999ed57c877081e9e1149ac3cd70268aa50f5e673edcec77dd to arohcpsvcdev.azurecr.io/ci-op-c5iz4fkw/pipeline:60efa6e90e760e999ed57c877081e9e1149ac3cd70268aa50f5e673edcec77dd.\nThe image will still be available under it's original digest sha256:60efa6e90e760e999ed57c877081e9e1149ac3cd70268aa50f5e673edcec77dd in the target registry.\nError: Put \"https://arohcpsvcdev.azurecr.io/v2/ci-op-c5iz4fkw/pipeline/blobs/uploads/15743118-8363-4850-8421-567ab29be8bf?_nouploadcache=false&_state=yL6O1Mkyse92xO6o0N7_ktVbhzUvvfcwZXvcxCNKiY17Ik5hbWUiOiJjaS1vcC1jNWl6NGZrdy9waXBlbGluZSIsIlVVSUQiOiIxNTc0MzExOC04MzYzLTQ4NTAtODQyMS01NjdhYjI5YmU4YmYiLCJPZmZzZXQiOjAsIlN0YXJ0ZWRBdCI6IjIwMjYtMDMtMjBUMTU6MTA6MTcuMzU3NTk1Mjk5WiJ9&digest=sha256%3A78a154f385e7d87fb9adfc808c2f030191c9b14cb2f9df801a6546418c3a47f9\": write tcp 10.130.136.56:50570->20.62.128.0:443: use of closed network connection\n exit status 1"

time=2026-03-20T15:04:46.129Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=global step=mirror-image description="Step mirror-image\n  Kind: ImageMirror\n  From {<nil> frontend.image.registry <nil>}:{<nil> frontend.image.repository <nil>}@{<nil> frontend.image.digest <nil>} to {<nil> acr.svc.name <nil>}\n"
time=2026-03-20T15:07:28.935Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=global step=mirror-image err="error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\nSetting up registry authentication for CI source registry.\ninfo: Using registry public hostname registry.build02.ci.openshift.org\nSaved credentials for registry.build02.ci.openshift.org into /tmp/tmp.tGfzCZRWRe/containers/auth.json\nLogging into target ACR arohcpsvcdev.\nLogin Succeeded\nMirroring image registry.build02.ci.openshift.org/ci-op-ddghffln/pipeline@sha256:5e8699aeeb3a8491206ef4a364616b18c4c964d854a38ebf8f1ac985dbffbb13 to arohcpsvcdev.azurecr.io/ci-op-ddghffln/pipeline:5e8699aeeb3a8491206ef4a364616b18c4c964d854a38ebf8f1ac985dbffbb13.\nThe image will still be available under it's original digest sha256:5e8699aeeb3a8491206ef4a364616b18c4c964d854a38ebf8f1ac985dbffbb13 in the target registry.\nError: Post \"https://arohcpsvcdev.azurecr.io/v2/ci-op-ddghffln/pipeline/blobs/uploads/\": EOF\n exit status 1"

time=2026-03-20T14:49:43.292Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=global step=mirror-image description="Step mirror-image\n  Kind: ImageMirror\n  From {<nil> backend.image.registry <nil>}:{<nil> backend.image.repository <nil>}@{<nil> backend.image.digest <nil>} to {<nil> acr.svc.name <nil>}\n"
time=2026-03-20T14:49:55.346Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=global step=mirror-image err="error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\nSetting up registry authentication for CI source registry.\ninfo: Using registry public hostname registry.build02.ci.openshift.org\nSaved credentials for registry.build02.ci.openshift.org into /tmp/tmp.6nYQLImg11/containers/auth.json\nLogging into target ACR arohcpsvcdev.\nFailed to log in to ACR arohcpsvcdev: ERROR: Could not connect to the registry login server 'arohcpsvcdev.azurecr.io'. Please verify that the registry exists and the URL 'https://arohcpsvcdev.azurecr.io/v2/' is reachable from your environment.\nTry running 'az acr check-health -n arohcpsvcdev --yes' to diagnose this issue.\n exit status 1"

fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:281]: Timed out after 30.001s.
Expected success, but got an error:
    <*fmt.wrapError | 0xc000b6e440>: 
    read access verification failed: get pods -n kube-system: pods is forbidden: User "system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652" cannot list resource "pods" in API group "" in the namespace "kube-system"
    get configmaps -n kube-system: configmaps is forbidden: User "system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652" cannot list resource "configmaps" in API group "" in the namespace "kube-system"
    {
        msg: "read access verification failed: get pods -n kube-system: pods is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"pods\" in API group \"\" in the namespace \"kube-system\"\nget configmaps -n kube-system: configmaps is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"configmaps\" in API group \"\" in the namespace \"kube-system\"",
        err: <*errors.joinError | 0xc0002110f8>{
            errs: [
                <*fmt.wrapError | 0xc000b6e160>{
                    msg: "get pods -n kube-system: pods is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"pods\" in API group \"\" in the namespace \"kube-system\"",
                    err: <*errors.StatusError | 0xc000b1d2c0>{
                        ErrStatus: {
                            TypeMeta: {Kind: "", APIVersion: ""},
                            ListMeta: {
                                SelfLink: "",
                                ResourceVersion: "",
                                Continue: "",
                                RemainingItemCount: nil,
                            },
                            Status: "Failure",
                            Message: "pods is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"pods\" in API group \"\" in the namespace \"kube-system\"",
                            Reason: "Forbidden",
                            Details: {Name: "", Group: "", Kind: "pods", UID: "", Causes: nil, RetryAfterSeconds: 0},
                            Code: 403,
                        },
                    },
                },
                <*fmt.wrapError | 0xc000b6e3e0>{
                    msg: "get configmaps -n kube-system: configmaps is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"configmaps\" in API group \"\" in the namespace \"kube-system\"",
                    err: <*errors.StatusError | 0xc000b1d540>{
                        ErrStatus: {
                            TypeMeta: {Kind: "", APIVersion: ""},
                            ListMeta: {
                                SelfLink: "",
                                ResourceVersion: "",
                                Continue: "",
                                RemainingItemCount: nil,
                            },
                            Status: "Failure",
                            Message: "configmaps is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"configmaps\" in API group \"\" in the namespace \"kube-system\"",
                            Reason: "Forbidden",
                            Details: {Name: "", Group: "", Kind: "configmaps", UID: "", Causes: nil, RetryAfterSeconds: 0},
                            Code: 403,
                        },
                    },
                },
            ],
        },
    }
fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:281]: Timed out after 30.001s.
Expected success, but got an error:
    <*fmt.wrapError | 0xc000b6e440>: 
    read access verification failed: get pods -n kube-system: pods is forbidden: User "system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652" cannot list resource "pods" in API group "" in the namespace "kube-system"
    get configmaps -n kube-system: configmaps is forbidden: User "system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652" cannot list resource "configmaps" in API group "" in the namespace "kube-system"
    {
        msg: "read access verification failed: get pods -n kube-system: pods is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"pods\" in API group \"\" in the namespace \"kube-system\"\nget configmaps -n kube-system: configmaps is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"configmaps\" in API group \"\" in the namespace \"kube-system\"",
        err: <*errors.joinError | 0xc0002110f8>{
            errs: [
                <*fmt.wrapError | 0xc000b6e160>{
                    msg: "get pods -n kube-system: pods is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"pods\" in API group \"\" in the namespace \"kube-system\"",
                    err: <*errors.StatusError | 0xc000b1d2c0>{
                        ErrStatus: {
                            TypeMeta: {Kind: "", APIVersion: ""},
                            ListMeta: {
                                SelfLink: "",
                                ResourceVersion: "",
                                Continue: "",
                                RemainingItemCount: nil,
                            },
                            Status: "Failure",
                            Message: "pods is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"pods\" in API group \"\" in the namespace \"kube-system\"",
                            Reason: "Forbidden",
                            Details: {Name: "", Group: "", Kind: "pods", UID: "", Causes: nil, RetryAfterSeconds: 0},
                            Code: 403,
                        },
                    },
                },
                <*fmt.wrapError | 0xc000b6e3e0>{
                    msg: "get configmaps -n kube-system: configmaps is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"configmaps\" in API group \"\" in the namespace \"kube-system\"",
                    err: <*errors.StatusError | 0xc000b1d540>{
                        ErrStatus: {
                            TypeMeta: {Kind: "", APIVersion: ""},
                            ListMeta: {
                                SelfLink: "",
                                ResourceVersion: "",
                                Continue: "",
                                RemainingItemCount: nil,
                            },
                            Status: "Failure",
                            Message: "configmaps is forbidden: User \"system:sre-break-glass:c209f8df-52ae-48fb-98ea-380f58b04652\" cannot list resource \"configmaps\" in API group \"\" in the namespace \"kube-system\"",
                            Reason: "Forbidden",
                            Details: {Name: "", Group: "", Kind: "configmaps", UID: "", Causes: nil, RetryAfterSeconds: 0},
                            Code: 403,
                        },
                    },
                },
            ],
        },
    }

time=2026-03-20T13:55:23.479Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=global step=mirror-image description="Step mirror-image\n  Kind: ImageMirror\n  From {<nil> maestro.image.registry <nil>}:{<nil> maestro.image.repository <nil>}@{<nil> maestro.image.digest <nil>} to {<nil> acr.svc.name <nil>}\n"
time=2026-03-20T13:55:47.157Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=global step=mirror-image err="error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\nLogging into target ACR arohcpsvcdev.\nFailed to log in to ACR arohcpsvcdev: ERROR: The command failed with an unexpected error. Here is the traceback:\nERROR: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\"))\nTraceback (most recent call last):\n  File \"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\", line 204, in _new_conn\n    sock = connection.create_connection(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/urllib3/util/connection.py\", line 60, in create_connection\n    for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):\n               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/python3.12/socket.py\", line 963, in getaddrinfo\n    for res in _socket.getaddrinfo(host, port, family, type, proto, flags):\n               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nsocket.gaierror: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File \"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 787, in urlopen\n    response = self._make_request(\n               ^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 488, in _make_request\n    raise new_e\n  File \"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 464, in _make_request\n    self._validate_conn(conn)\n  File \"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 1093, in _validate_conn\n    conn.connect()\n  File \"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\", line 759, in connect\n    self.sock = sock = self._new_conn()\n                       ^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\", line 211, in _new_conn\n    raise NameResolutionError(self.host, self, e) from e\nurllib3.exceptions.NameResolutionError: HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File \"/usr/lib64/az/lib/python3.12/site-packages/requests/adapters.py\", line 667, in send\n    resp = conn.urlopen(\n           ^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 841, in urlopen\n    retries = retries.increment(\n              ^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/urllib3/util/retry.py\", line 535, in increment\n    raise MaxRetryError(_pool, url, reason) from reason  # type: ignore[arg-type]\n    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nurllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\"))\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File \"/usr/lib64/az/lib/python3.12/site-packages/knack/cli.py\", line 233, in invoke\n    cmd_result = self.invocation.execute(args)\n                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\", line 682, in execute\n    raise ex\n  File \"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\", line 812, in _run_jobs_serially\n    results.append(self._run_job(expanded_arg, cmd_copy))\n                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\", line 781, in _run_job\n    result = cmd_copy(params)\n             ^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\", line 336, in __call__\n    return self.handler(*args, **kwargs)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/command_operation.py\", line 120, in handler\n    return op(**command_args)\n           ^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/custom.py\", line 294, in acr_login\n    login_server, _, password = get_login_credentials(\n                                ^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\", line 505, in get_login_credentials\n    return _get_credentials(cmd,\n           ^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\", line 449, in _get_credentials\n    return login_server, EMPTY_GUID, _get_aad_token(cli_ctx,\n                                     ^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\", line 276, in _get_aad_token\n    token_params = _handle_challenge_phase(\n                   ^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\", line 100, in _handle_challenge_phase\n    challenge = requests.get(request_url, verify=not should_disable_connection_verify())\n                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/requests/api.py\", line 73, in get\n    return request(\"get\", url, params=params, **kwargs)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/requests/api.py\", line 59, in request\n    return session.request(method=method, url=url, **kwargs)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/requests/sessions.py\", line 589, in request\n    resp = self.send(prep, **send_kwargs)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/requests/sessions.py\", line 703, in send\n    r = adapter.send(request, **kwargs)\n        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/requests/adapters.py\", line 700, in send\n    raise ConnectionError(e, request=request)\nrequests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\"))\nTo check existing issues, please visit: https://github.com/Azure/azure-cli/issues\n exit status 1"

fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:74]: Failed to set up port-forward
Unexpected error:
    <*fmt.wrapError | 0xc0004bd080>: 
    port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.10:10250, code 500: 500 Internal Server Error
    {
        msg: "port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.10:10250, code 500: 500 Internal Server Error",
        err: <*errors.errorString | 0xc0008ef350>{
            s: "error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.10:10250, code 500: 500 Internal Server Error",
        },
    }
occurred
fail [github.com/Azure/ARO-HCP/test/e2e/exporter_metrics.go:74]: Failed to set up port-forward
Unexpected error:
    <*fmt.wrapError | 0xc0004bd080>: 
    port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.10:10250, code 500: 500 Internal Server Error
    {
        msg: "port-forward failed: error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.10:10250, code 500: 500 Internal Server Error",
        err: <*errors.errorString | 0xc0008ef350>{
            s: "error upgrading connection: error dialing backend: proxy error from localhost:9443 while dialing 10.128.8.10:10250, code 500: 500 Internal Server Error",
        },
    }
occurred

fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:245]: Unexpected error:
    <*errors.joinError | 0xc00102cae0>: 
    found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-8x9hrr
    {
        errs: [
            <*errors.errorString | 0xc0015bede0>{
                s: "found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-8x9hrr",
            },
        ],
    }
occurred
fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:245]: Unexpected error:
    <*errors.joinError | 0xc00102cae0>: 
    found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-8x9hrr
    {
        errs: [
            <*errors.errorString | 0xc0015bede0>{
                s: "found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-8x9hrr",
            },
        ],
    }
occurred

time=2026-03-20T07:16:55.672Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image description="Step mirror-image\n  Kind: ImageMirror\n  From {<nil> secretSyncController.image.registry <nil>}:{<nil> secretSyncController.image.repository <nil>}@{<nil> secretSyncController.image.digest <nil>} to {<nil> acr.svc.name <nil>}\n"
time=2026-03-20T07:17:18.851Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image err="error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\nFetch pull secret for source registry registry.k8s.io from arohcpdev-global KV.\nLogging into target ACR arohcpsvcdev.\nFailed to log in to ACR arohcpsvcdev: ERROR: Could not connect to the registry login server 'arohcpsvcdev.azurecr.io'. Please verify that the registry exists and the URL 'https://arohcpsvcdev.azurecr.io/v2/' is reachable from your environment.\nTry running 'az acr check-health -n arohcpsvcdev --yes' to diagnose this issue.\n exit status 1"

time=2026-03-20T07:08:18.552Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=global step=mirror-operator-image description="Step mirror-operator-image\n  Kind: ImageMirror\n  From {<nil> routeMonitorOperator.operatorImage.registry <nil>}:{<nil> routeMonitorOperator.operatorImage.repository <nil>}@{<nil> routeMonitorOperator.operatorImage.digest <nil>} to {<nil> acr.svc.name <nil>}\n"
time=2026-03-20T07:08:32.083Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.RouteMonitorOperator resourceGroup=global step=mirror-operator-image err="error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\nLogging into target ACR arohcpsvcdev.\nFailed to log in to ACR arohcpsvcdev: ERROR: The command failed with an unexpected error. Here is the traceback:\nERROR: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\"))\nTraceback (most recent call last):\n  File \"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\", line 204, in _new_conn\n    sock = connection.create_connection(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/urllib3/util/connection.py\", line 60, in create_connection\n    for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):\n               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/python3.12/socket.py\", line 963, in getaddrinfo\n    for res in _socket.getaddrinfo(host, port, family, type, proto, flags):\n               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nsocket.gaierror: [Errno -2] Name or service not known\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File \"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 787, in urlopen\n    response = self._make_request(\n               ^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 488, in _make_request\n    raise new_e\n  File \"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 464, in _make_request\n    self._validate_conn(conn)\n  File \"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 1093, in _validate_conn\n    conn.connect()\n  File \"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\", line 759, in connect\n    self.sock = sock = self._new_conn()\n                       ^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connection.py\", line 211, in _new_conn\n    raise NameResolutionError(self.host, self, e) from e\nurllib3.exceptions.NameResolutionError: HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File \"/usr/lib64/az/lib/python3.12/site-packages/requests/adapters.py\", line 667, in send\n    resp = conn.urlopen(\n           ^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/urllib3/connectionpool.py\", line 841, in urlopen\n    retries = retries.increment(\n              ^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/urllib3/util/retry.py\", line 535, in increment\n    raise MaxRetryError(_pool, url, reason) from reason  # type: ignore[arg-type]\n    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nurllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\"))\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File \"/usr/lib64/az/lib/python3.12/site-packages/knack/cli.py\", line 233, in invoke\n    cmd_result = self.invocation.execute(args)\n                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\", line 682, in execute\n    raise ex\n  File \"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\", line 812, in _run_jobs_serially\n    results.append(self._run_job(expanded_arg, cmd_copy))\n                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\", line 781, in _run_job\n    result = cmd_copy(params)\n             ^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/__init__.py\", line 336, in __call__\n    return self.handler(*args, **kwargs)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/core/commands/command_operation.py\", line 120, in handler\n    return op(**command_args)\n           ^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/custom.py\", line 294, in acr_login\n    login_server, _, password = get_login_credentials(\n                                ^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\", line 505, in get_login_credentials\n    return _get_credentials(cmd,\n           ^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\", line 449, in _get_credentials\n    return login_server, EMPTY_GUID, _get_aad_token(cli_ctx,\n                                     ^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\", line 276, in _get_aad_token\n    token_params = _handle_challenge_phase(\n                   ^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/azure/cli/command_modules/acr/_docker_utils.py\", line 100, in _handle_challenge_phase\n    challenge = requests.get(request_url, verify=not should_disable_connection_verify())\n                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/requests/api.py\", line 73, in get\n    return request(\"get\", url, params=params, **kwargs)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/requests/api.py\", line 59, in request\n    return session.request(method=method, url=url, **kwargs)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/requests/sessions.py\", line 589, in request\n    resp = self.send(prep, **send_kwargs)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/requests/sessions.py\", line 703, in send\n    r = adapter.send(request, **kwargs)\n        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/usr/lib64/az/lib/python3.12/site-packages/requests/adapters.py\", line 700, in send\n    raise ConnectionError(e, request=request)\nrequests.exceptions.ConnectionError: HTTPSConnectionPool(host='arohcpsvcdev.azurecr.io', port=443): Max retries exceeded with url: /v2/ (Caused by NameResolutionError(\"HTTPSConnection(host='arohcpsvcdev.azurecr.io', port=443): Failed to resolve 'arohcpsvcdev.azurecr.io' ([Errno -2] Name or service not known)\"))\nTo check existing issues, please visit: https://github.com/Azure/azure-cli/issues\n exit status 1"

time=2026-03-20T06:21:29.525Z level=INFO msg="Running step." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy description="Step deploy\n Kind: Helm\n"
time=2026-03-20T06:21:30.204Z level=INFO msg="Resolved input values." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy values="map[adminApi:map[namespace:aro-hcp-admin-api serviceAccount:admin-api] adminApiHost:admin.j4488576.hcpsvc.osadev.cloud apiBindPort:8443 audit:map[connectSocket:false] azure:map[clientId:48b75c14-ea56-4787-9d59-4037fb62f6d9 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[databaseName:arohcpprow-rp-j4488576 databaseUrl:https://arohcpprow-rp-j4488576.documents.azure.com:443/ frontendMiClientId:a16fb73c-60de-4f29-a20d-f30ea1e20ea2 location:westus3] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:frontend-cert-prow-j4488576] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-mjqmwc9t/pipeline@sha256:866c17aa81505a50b1a480b0165400a0a61d5dab459c90451430e1843d948f43 replicas:2 zoneCount:4] exitOnPanic:true mise:map[deploy:false policyLabel:ARM Policy] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-mjqmwc9t/pipeline:pull workloadIdentityClientId:6a650699-8993-4e92-a28a-74e8710f9988 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:frontend workloadIdentityClientId:a16fb73c-60de-4f29-a20d-f30ea1e20ea2 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tracing:map[address:http://ingest.observability:4318 exporter:otlp] virtualService:map[host:rp.j4488576.hcpsvc.osadev.cloud]]"
time=2026-03-20T06:21:30.205Z level=INFO msg="Applying namespaces." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy
time=2026-03-20T06:21:30.548Z level=INFO msg="Namespace applied successfully." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy namespace=aro-hcp
time=2026-03-20T06:21:30.844Z level=INFO msg="Rolling out Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy dryRun=false
time=2026-03-20T06:21:30.844Z level=INFO msg="Searching for release history..." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy
time=2026-03-20T06:21:30.956Z level=INFO msg="No release history found, running the first release..." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy
time=2026-03-20T06:26:40.307Z level=ERROR msg="Failed to roll out the Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy err="resource not ready, name: aro-hcp-frontend, kind: Deployment, status: InProgress\ncontext deadline exceeded"
time=2026-03-20T06:26:40.307Z level=INFO msg="Finished deploying Helm release." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy
time=2026-03-20T06:26:40.307Z level=INFO msg="Running inline diagnostics." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy
time=2026-03-20T06:26:43.871Z level=INFO msg="Determined release status." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy release=aro-hcp-frontend-dev namespace=aro-hcp status=failed description="Release \"aro-hcp-frontend-dev\" failed: resource not ready, name: aro-hcp-frontend, kind: Deployment, status: InProgress\ncontext deadline exceeded" values="map[adminApi:map[namespace:aro-hcp-admin-api serviceAccount:admin-api] adminApiHost:admin.j4488576.hcpsvc.osadev.cloud apiBindPort:8443 audit:map[connectSocket:false] azure:map[clientId:48b75c14-ea56-4787-9d59-4037fb62f6d9 tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[databaseName:arohcpprow-rp-j4488576 databaseUrl:https://arohcpprow-rp-j4488576.documents.azure.com:443/ frontendMiClientId:a16fb73c-60de-4f29-a20d-f30ea1e20ea2 location:westus3] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:frontend-cert-prow-j4488576] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-mjqmwc9t/pipeline@sha256:866c17aa81505a50b1a480b0165400a0a61d5dab459c90451430e1843d948f43 replicas:2 zoneCount:4] exitOnPanic:true mise:map[deploy:false policyLabel:ARM Policy] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-mjqmwc9t/pipeline:pull workloadIdentityClientId:6a650699-8993-4e92-a28a-74e8710f9988 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:frontend workloadIdentityClientId:a16fb73c-60de-4f29-a20d-f30ea1e20ea2 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tracing:map[address:http://ingest.observability:4318 exporter:otlp] virtualService:map[host:rp.j4488576.hcpsvc.osadev.cloud]]"
time=2026-03-20T06:26:43.874Z level=INFO msg="Kube-events kusto link for troubleshooting:" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy url="https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/5yUTY/aPhDG7/kUFieQ8Io/+xcHqj3sS9VDtRVSqx5a9eDYQ5jFmYnGEygVH74KLyVQFtE95vHzGz8eTSaLoEYgcS0ekrkzwalTl0focv4CXj8ihXFSQSr6Zit9ciX8kWABtFFS5fxe7n3PjDGm8xm8gE6EFxhAHqNLqdM3nakwKVCwyVfNt5sni0mRLVIh0Jj2vCzQwzMTKsvGKmxnvrL7Ei1tD31F0drFHduGFukSN+HwhEnqSpHpoQ4F6JU37q66955r0vYLz5gfmaZYPLvqqBN+o75e+9+SnHPbElTQpzPUvZdJHeMDUkAqjnJVdYw2PxyckrXOWPCXa3o24Yh+tXHFyEvrQolkXYVW+VLoi0V2qd/ME+vsDdlPxvEv+INTWLpVu9HFVrLwU0HIxYuT/QRV5FUJdO2MHQOHuQFRnKJ3ClZgKpBmIBdvngBI83QgbTBkunZash/vsnmdgxAopPfNj5+ytVnOQMAolpDUlZXJQZcAZLrBKTRytzMcDEd2cGuHgy+D0Xj43/h28K3TMzc35lXPaPx/4+lla/PCSGaOFO6QCKS1rpjMYUu119PpXsrWphJuTg9B+6bisDVfV6RvIhd9U0JKrmhKsgQQk69ajw+Q/O8AAAD//xkTKTBUBQAA"
time=2026-03-20T06:26:43.876Z level=INFO msg="Deployment complete." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy
time=2026-03-20T06:26:43.876Z level=ERROR msg="Step errored." serviceGroup=Microsoft.Azure.ARO.HCP.RP.Frontend resourceGroup=service step=deploy err="error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-frontend, kind: Deployment, status: InProgress\ncontext deadline exceeded"