{"meta":{"date":"2026-02-25","timezone":"UTC","generated_at":"2026-11-02T03:03:31Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":81,"failed_runs":49,"runs_with_occurrences":49,"runs_with_failure_pattern_matches":42,"runs_with_unmatched_occurrences":7,"runs_without_occurrence_rows":32,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4196/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026807189007503360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4196,"pr_state":"closed","pr_sha":"5a03840502dfd5db86eca68bfb8e212bff6f84ab","final_merged_sha":"b541ca7e953f8e1929cb1cc01938e54152b9c8e2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T23:51:04Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"020161546f086901186287353f8e4f363522f99cd7d99b3a74c3464cabc1b538","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4196/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026807189007503360","occurred_at":"2026-02-25T23:51:04Z","failed_at":"e2e","signature_id":"75f97c0bd0ffa820479cf6d357b2c64b1f4cfc2eea487aa97b8ca711898cf7c1","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000f1e780\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-jsnx7r\" to finish creating: GET https://rp.j7503360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/225ca079-6d7a-4934-af01-b741ca97f4ee\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/225ca079-6d7a-4934-af01-b741ca97f4ee\",\n      \"name\": \"225ca079-6d7a-4934-af01-b741ca97f4ee\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-26T00:54:19.286966114Z\",\n      \"endTime\": \"2026-02-26T00:57:01.705617058Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"GET https://arohcpprowoidcj7503360.blob.core.windows.net/$web\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\nERROR CODE: AuthorizationPermissionMismatch\\n--------------------------------------------------------------------------------\\n\\u003c?xml version=\\\"1.0\\\" encoding=\\\"utf-8\\\"?\\u003e\\u003cError\\u003e\\u003cCode\\u003eAuthorizationPermissionMismatch\\u003c/Code\\u003e\\u003cMessage\\u003eThis request is not authorized to perform this operation using this permission.\\nRequestId:d028f085-801e-00ad-72ba-a6c4ef000000\\nTime:2026-02-26T00:56:09.4065931Z\\u003c/Message\\u003e\\u003c/Error\\u003e\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster basic-cluster: failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-jsnx7r\\\" to finish creating: GET https://rp.j7503360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/225ca079-6d7a-4934-af01-b741ca97f4ee\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/225ca079-6d7a-4934-af01-b741ca97f4ee\\\",\\n  \\\"name\\\": \\\"225ca079-6d7a-4934-af01-b741ca97f4ee\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-26T00:54:19.286966114Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-26T00:57:01.705617058Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"GET https://arohcpprowoidcj7503360.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:d028f085-801e-00ad-72ba-a6c4ef000000\\\\nTime:2026-02-26T00:56:09.4065931Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000f1e760\u003e{\n            msg: \"failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-jsnx7r\\\" to finish creating: GET https://rp.j7503360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/225ca079-6d7a-4934-af01-b741ca97f4ee\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/225ca079-6d7a-4934-af01-b741ca97f4ee\\\",\\n  \\\"name\\\": \\\"225ca079-6d7a-4934-af01-b741ca97f4ee\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-26T00:54:19.286966114Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-26T00:57:01.705617058Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"GET https://arohcpprowoidcj7503360.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:d028f085-801e-00ad-72ba-a6c4ef000000\\\\nTime:2026-02-26T00:56:09.4065931Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n      ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000f1e780\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-jsnx7r\" to finish creating: GET https://rp.j7503360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/225ca079-6d7a-4934-af01-b741ca97f4ee\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/225ca079-6d7a-4934-af01-b741ca97f4ee\",\n      \"name\": \"225ca079-6d7a-4934-af01-b741ca97f4ee\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-26T00:54:19.286966114Z\",\n      \"endTime\": \"2026-02-26T00:57:01.705617058Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"GET https://arohcpprowoidcj7503360.blob.core.windows.net/$web\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\nERROR CODE: AuthorizationPermissionMismatch\\n--------------------------------------------------------------------------------\\n\\u003c?xml version=\\\"1.0\\\" encoding=\\\"utf-8\\\"?\\u003e\\u003cError\\u003e\\u003cCode\\u003eAuthorizationPermissionMismatch\\u003c/Code\\u003e\\u003cMessage\\u003eThis request is not authorized to perform this operation using this permission.\\nRequestId:d028f085-801e-00ad-72ba-a6c4ef000000\\nTime:2026-02-26T00:56:09.4065931Z\\u003c/Message\\u003e\\u003c/Error\\u003e\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster basic-cluster: failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-jsnx7r\\\" to finish creating: GET https://rp.j7503360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/225ca079-6d7a-4934-af01-b741ca97f4ee\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/225ca079-6d7a-4934-af01-b741ca97f4ee\\\",\\n  \\\"name\\\": \\\"225ca079-6d7a-4934-af01-b741ca97f4ee\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-26T00:54:19.286966114Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-26T00:57:01.705617058Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"GET https://arohcpprowoidcj7503360.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:d028f085-801e-00ad-72ba-a6c4ef000000\\\\nTime:2026-02-26T00:56:09.4065931Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000f1e760\u003e{\n            msg: \"failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-jsnx7r\\\" to finish creating: GET https://rp.j7503360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/225ca079-6d7a-4934-af01-b741ca97f4ee\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/225ca079-6d7a-4934-af01-b741ca97f4ee\\\",\\n  \\\"name\\\": \\\"225ca079-6d7a-4934-af01-b741ca97f4ee\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-26T00:54:19.286966114Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-26T00:57:01.705617058Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"GET https://arohcpprowoidcj7503360.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:d028f085-801e-00ad-72ba-a6c4ef000000\\\\nTime:2026-02-26T00:56:09.4065931Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n      ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"eea64eddec12ea7a6221c8fd40bb79ac7f62074284efa8adfa6285e8d21d2b1a","failure_pattern":"ERROR CODE: InternalServerError; detail code AuthorizationPermissionMismatch; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5ce0ed179a97868d74074768dee43081e67c5e7eecd1c5746b40b262c894a95e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4196/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026807189007503360","occurred_at":"2026-02-25T23:51:04Z","failed_at":"e2e","signature_id":"60a35d52304b020daa17d3b787466e89185d9c464451b67740a30171227b5627","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000d74d40\u003e: \n    failed to create HCP cluster ea-listhdnd74: failed waiting for cluster=\"ea-listhdnd74\" in resourcegroup=\"ea-list-cm8692\" to finish creating: GET https://rp.j7503360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19a5ca75-3490-45ee-b725-bdab9d1c2f87\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19a5ca75-3490-45ee-b725-bdab9d1c2f87\",\n      \"name\": \"19a5ca75-3490-45ee-b725-bdab9d1c2f87\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-26T00:54:08.817035844Z\",\n      \"endTime\": \"2026-02-26T00:57:01.558964391Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"GET https://arohcpprowoidcj7503360.blob.core.windows.net/$web\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\nERROR CODE: AuthorizationPermissionMismatch\\n--------------------------------------------------------------------------------\\n\\u003c?xml version=\\\"1.0\\\" encoding=\\\"utf-8\\\"?\\u003e\\u003cError\\u003e\\u003cCode\\u003eAuthorizationPermissionMismatch\\u003c/Code\\u003e\\u003cMessage\\u003eThis request is not authorized to perform this operation using this permission.\\nRequestId:d028ec26-801e-00ad-74ba-a6c4ef000000\\nTime:2026-02-26T00:56:09.2186001Z\\u003c/Message\\u003e\\u003c/Error\\u003e\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-listhdnd74: failed waiting for cluster=\\\"ea-listhdnd74\\\" in resourcegroup=\\\"ea-list-cm8692\\\" to finish creating: GET https://rp.j7503360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19a5ca75-3490-45ee-b725-bdab9d1c2f87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19a5ca75-3490-45ee-b725-bdab9d1c2f87\\\",\\n  \\\"name\\\": \\\"19a5ca75-3490-45ee-b725-bdab9d1c2f87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-26T00:54:08.817035844Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-26T00:57:01.558964391Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"GET https://arohcpprowoidcj7503360.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:d028ec26-801e-00ad-74ba-a6c4ef000000\\\\nTime:2026-02-26T00:56:09.2186001Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000d74d20\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-listhdnd74\\\" in resourcegroup=\\\"ea-list-cm8692\\\" to finish creating: GET https://rp.j7503360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19a5ca75-3490-45ee-b725-bdab9d1c2f87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19a5ca75-3490-45ee-b725-bdab9d1c2f87\\\",\\n  \\\"name\\\": \\\"19a5ca75-3490-45ee-b725-bdab9d1c2f87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-26T00:54:08.817035844Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-26T00:57:01.558964391Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"GET https://arohcpprowoidcj7503360.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:d028ec26-801e-00ad-74ba-a6c4ef000000\\\\nTime:2026-02-26T00:56:09.2186001Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0x...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000d74d40\u003e: \n    failed to create HCP cluster ea-listhdnd74: failed waiting for cluster=\"ea-listhdnd74\" in resourcegroup=\"ea-list-cm8692\" to finish creating: GET https://rp.j7503360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19a5ca75-3490-45ee-b725-bdab9d1c2f87\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19a5ca75-3490-45ee-b725-bdab9d1c2f87\",\n      \"name\": \"19a5ca75-3490-45ee-b725-bdab9d1c2f87\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-26T00:54:08.817035844Z\",\n      \"endTime\": \"2026-02-26T00:57:01.558964391Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"GET https://arohcpprowoidcj7503360.blob.core.windows.net/$web\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\nERROR CODE: AuthorizationPermissionMismatch\\n--------------------------------------------------------------------------------\\n\\u003c?xml version=\\\"1.0\\\" encoding=\\\"utf-8\\\"?\\u003e\\u003cError\\u003e\\u003cCode\\u003eAuthorizationPermissionMismatch\\u003c/Code\\u003e\\u003cMessage\\u003eThis request is not authorized to perform this operation using this permission.\\nRequestId:d028ec26-801e-00ad-74ba-a6c4ef000000\\nTime:2026-02-26T00:56:09.2186001Z\\u003c/Message\\u003e\\u003c/Error\\u003e\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-listhdnd74: failed waiting for cluster=\\\"ea-listhdnd74\\\" in resourcegroup=\\\"ea-list-cm8692\\\" to finish creating: GET https://rp.j7503360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19a5ca75-3490-45ee-b725-bdab9d1c2f87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19a5ca75-3490-45ee-b725-bdab9d1c2f87\\\",\\n  \\\"name\\\": \\\"19a5ca75-3490-45ee-b725-bdab9d1c2f87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-26T00:54:08.817035844Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-26T00:57:01.558964391Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"GET https://arohcpprowoidcj7503360.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:d028ec26-801e-00ad-74ba-a6c4ef000000\\\\nTime:2026-02-26T00:56:09.2186001Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000d74d20\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-listhdnd74\\\" in resourcegroup=\\\"ea-list-cm8692\\\" to finish creating: GET https://rp.j7503360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19a5ca75-3490-45ee-b725-bdab9d1c2f87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19a5ca75-3490-45ee-b725-bdab9d1c2f87\\\",\\n  \\\"name\\\": \\\"19a5ca75-3490-45ee-b725-bdab9d1c2f87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-26T00:54:08.817035844Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-26T00:57:01.558964391Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"GET https://arohcpprowoidcj7503360.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:d028ec26-801e-00ad-74ba-a6c4ef000000\\\\nTime:2026-02-26T00:56:09.2186001Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0x...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"eea64eddec12ea7a6221c8fd40bb79ac7f62074284efa8adfa6285e8d21d2b1a","failure_pattern":"ERROR CODE: InternalServerError; detail code AuthorizationPermissionMismatch; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f0a5ba50f5832b8786ab3df9084a788e89b0aa84f7f87b840f579d7db61af338","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4196/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026807189007503360","occurred_at":"2026-02-25T23:51:04Z","failed_at":"e2e","signature_id":"0e00eabb86ab868f815fe8cd1c91c83cdd1d09e386be73cc5f9d4bf8ae0054f6","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000721080\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-fc4gzl\" to finish creating: GET https://rp.j7503360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f2261cdb-1994-4cbf-afe2-37e4ffb6f562\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f2261cdb-1994-4cbf-afe2-37e4ffb6f562\",\n      \"name\": \"f2261cdb-1994-4cbf-afe2-37e4ffb6f562\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-26T00:54:12.525607564Z\",\n      \"endTime\": \"2026-02-26T00:57:01.857734749Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"PUT https://arohcpprowoidcj7503360.blob.core.windows.net/$web/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/2omknd2gh99pacnb5gakhposl180tdg0/openid/v1/jwks\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\nERROR CODE: AuthorizationPermissionMismatch\\n--------------------------------------------------------------------------------\\n\\u003c?xml version=\\\"1.0\\\" encoding=\\\"utf-8\\\"?\\u003e\\u003cError\\u003e\\u003cCode\\u003eAuthorizationPermissionMismatch\\u003c/Code\\u003e\\u003cMessage\\u003eThis request is not authorized to perform this operation using this permission.\\nRequestId:d028f0fb-801e-00ad-58ba-a6c4ef000000\\nTime:2026-02-26T00:56:09.6363965Z\\u003c/Message\\u003e\\u003c/Error\\u003e\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-fc4gzl\\\" to finish creating: GET https://rp.j7503360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f2261cdb-1994-4cbf-afe2-37e4ffb6f562\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f2261cdb-1994-4cbf-afe2-37e4ffb6f562\\\",\\n  \\\"name\\\": \\\"f2261cdb-1994-4cbf-afe2-37e4ffb6f562\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-26T00:54:12.525607564Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-26T00:57:01.857734749Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"PUT https://arohcpprowoidcj7503360.blob.core.windows.net/$web/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/2omknd2gh99pacnb5gakhposl180tdg0/openid/v1/jwks\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:d028f0fb-801e-00ad-58ba-a6c4ef000000\\\\nTime:2026-02-26T00:56:09.6363965Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000721060\u003e{\n            msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-fc4gzl\\\" to finish creating: GET https://rp.j7503360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f2261cdb-1994-4cbf-afe2-37e4ffb6f562\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f2261cdb-1994-4cbf-afe2-37e4ffb6f562\\\",\\n  \\\"name\\\": \\\"f2261cdb-1994-4cbf-afe2-37e4ffb6f562\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-26T00:54:12.525607564Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-26T00:57:01.857734749Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"PUT https://arohcpprowoidcj7503360.blob.core.windows.net/$web/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/2omknd2gh99pacnb5gakhposl180tdg0/openid/v1/jwks\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:d028f0fb-801e-00ad-58ba-a6c4ef000000\\\\nTime:2026-02-26T00:56:09.6363965Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n-------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000721080\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-fc4gzl\" to finish creating: GET https://rp.j7503360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f2261cdb-1994-4cbf-afe2-37e4ffb6f562\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f2261cdb-1994-4cbf-afe2-37e4ffb6f562\",\n      \"name\": \"f2261cdb-1994-4cbf-afe2-37e4ffb6f562\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-26T00:54:12.525607564Z\",\n      \"endTime\": \"2026-02-26T00:57:01.857734749Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"PUT https://arohcpprowoidcj7503360.blob.core.windows.net/$web/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/2omknd2gh99pacnb5gakhposl180tdg0/openid/v1/jwks\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\nERROR CODE: AuthorizationPermissionMismatch\\n--------------------------------------------------------------------------------\\n\\u003c?xml version=\\\"1.0\\\" encoding=\\\"utf-8\\\"?\\u003e\\u003cError\\u003e\\u003cCode\\u003eAuthorizationPermissionMismatch\\u003c/Code\\u003e\\u003cMessage\\u003eThis request is not authorized to perform this operation using this permission.\\nRequestId:d028f0fb-801e-00ad-58ba-a6c4ef000000\\nTime:2026-02-26T00:56:09.6363965Z\\u003c/Message\\u003e\\u003c/Error\\u003e\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-fc4gzl\\\" to finish creating: GET https://rp.j7503360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f2261cdb-1994-4cbf-afe2-37e4ffb6f562\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f2261cdb-1994-4cbf-afe2-37e4ffb6f562\\\",\\n  \\\"name\\\": \\\"f2261cdb-1994-4cbf-afe2-37e4ffb6f562\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-26T00:54:12.525607564Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-26T00:57:01.857734749Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"PUT https://arohcpprowoidcj7503360.blob.core.windows.net/$web/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/2omknd2gh99pacnb5gakhposl180tdg0/openid/v1/jwks\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:d028f0fb-801e-00ad-58ba-a6c4ef000000\\\\nTime:2026-02-26T00:56:09.6363965Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000721060\u003e{\n            msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-fc4gzl\\\" to finish creating: GET https://rp.j7503360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f2261cdb-1994-4cbf-afe2-37e4ffb6f562\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f2261cdb-1994-4cbf-afe2-37e4ffb6f562\\\",\\n  \\\"name\\\": \\\"f2261cdb-1994-4cbf-afe2-37e4ffb6f562\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-26T00:54:12.525607564Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-26T00:57:01.857734749Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"PUT https://arohcpprowoidcj7503360.blob.core.windows.net/$web/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/2omknd2gh99pacnb5gakhposl180tdg0/openid/v1/jwks\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:d028f0fb-801e-00ad-58ba-a6c4ef000000\\\\nTime:2026-02-26T00:56:09.6363965Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n-------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"eea64eddec12ea7a6221c8fd40bb79ac7f62074284efa8adfa6285e8d21d2b1a","failure_pattern":"ERROR CODE: InternalServerError; detail code AuthorizationPermissionMismatch; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f8ee08b7bc63c166c5f883ba6ca752c9eaa34f696312ebfa52871db537800ae4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4196/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026807189007503360","occurred_at":"2026-02-25T23:51:04Z","failed_at":"e2e","signature_id":"5e9a561c67b2f6edf8fa040ce1fb93ea4b2dbe1b92598eacc38243e953e80727","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009189e0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-xz7m8f\" to finish creating: GET https://rp.j7503360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1869cf79-9ebe-43a3-9d4f-06ed3bc51b03\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1869cf79-9ebe-43a3-9d4f-06ed3bc51b03\",\n      \"name\": \"1869cf79-9ebe-43a3-9d4f-06ed3bc51b03\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-26T00:54:29.822288857Z\",\n      \"endTime\": \"2026-02-26T00:57:02.314815127Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"GET https://arohcpprowoidcj7503360.blob.core.windows.net/$web\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\nERROR CODE: AuthorizationPermissionMismatch\\n--------------------------------------------------------------------------------\\n\\u003c?xml version=\\\"1.0\\\" encoding=\\\"utf-8\\\"?\\u003e\\u003cError\\u003e\\u003cCode\\u003eAuthorizationPermissionMismatch\\u003c/Code\\u003e\\u003cMessage\\u003eThis request is not authorized to perform this operation using this permission.\\nRequestId:d028f05f-801e-00ad-4eba-a6c4ef000000\\nTime:2026-02-26T00:56:09.3125409Z\\u003c/Message\\u003e\\u003c/Error\\u003e\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-xz7m8f\\\" to finish creating: GET https://rp.j7503360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1869cf79-9ebe-43a3-9d4f-06ed3bc51b03\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1869cf79-9ebe-43a3-9d4f-06ed3bc51b03\\\",\\n  \\\"name\\\": \\\"1869cf79-9ebe-43a3-9d4f-06ed3bc51b03\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-26T00:54:29.822288857Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-26T00:57:02.314815127Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"GET https://arohcpprowoidcj7503360.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:d028f05f-801e-00ad-4eba-a6c4ef000000\\\\nTime:2026-02-26T00:56:09.3125409Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000918980\u003e{\n            msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-xz7m8f\\\" to finish creating: GET https://rp.j7503360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1869cf79-9ebe-43a3-9d4f-06ed3bc51b03\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1869cf79-9ebe-43a3-9d4f-06ed3bc51b03\\\",\\n  \\\"name\\\": \\\"1869cf79-9ebe-43a3-9d4f-06ed3bc51b03\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-26T00:54:29.822288857Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-26T00:57:02.314815127Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"GET https://arohcpprowoidcj7503360.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:d028f05f-801e-00ad-4eba-a6c4ef000000\\\\nTime:2026-02-26T00:56:09.3125409Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n-----------------------------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009189e0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-xz7m8f\" to finish creating: GET https://rp.j7503360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1869cf79-9ebe-43a3-9d4f-06ed3bc51b03\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1869cf79-9ebe-43a3-9d4f-06ed3bc51b03\",\n      \"name\": \"1869cf79-9ebe-43a3-9d4f-06ed3bc51b03\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-26T00:54:29.822288857Z\",\n      \"endTime\": \"2026-02-26T00:57:02.314815127Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"GET https://arohcpprowoidcj7503360.blob.core.windows.net/$web\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\nERROR CODE: AuthorizationPermissionMismatch\\n--------------------------------------------------------------------------------\\n\\u003c?xml version=\\\"1.0\\\" encoding=\\\"utf-8\\\"?\\u003e\\u003cError\\u003e\\u003cCode\\u003eAuthorizationPermissionMismatch\\u003c/Code\\u003e\\u003cMessage\\u003eThis request is not authorized to perform this operation using this permission.\\nRequestId:d028f05f-801e-00ad-4eba-a6c4ef000000\\nTime:2026-02-26T00:56:09.3125409Z\\u003c/Message\\u003e\\u003c/Error\\u003e\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-xz7m8f\\\" to finish creating: GET https://rp.j7503360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1869cf79-9ebe-43a3-9d4f-06ed3bc51b03\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1869cf79-9ebe-43a3-9d4f-06ed3bc51b03\\\",\\n  \\\"name\\\": \\\"1869cf79-9ebe-43a3-9d4f-06ed3bc51b03\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-26T00:54:29.822288857Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-26T00:57:02.314815127Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"GET https://arohcpprowoidcj7503360.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:d028f05f-801e-00ad-4eba-a6c4ef000000\\\\nTime:2026-02-26T00:56:09.3125409Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000918980\u003e{\n            msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-xz7m8f\\\" to finish creating: GET https://rp.j7503360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1869cf79-9ebe-43a3-9d4f-06ed3bc51b03\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1869cf79-9ebe-43a3-9d4f-06ed3bc51b03\\\",\\n  \\\"name\\\": \\\"1869cf79-9ebe-43a3-9d4f-06ed3bc51b03\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-26T00:54:29.822288857Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-26T00:57:02.314815127Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"GET https://arohcpprowoidcj7503360.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:d028f05f-801e-00ad-4eba-a6c4ef000000\\\\nTime:2026-02-26T00:56:09.3125409Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n-----------------------------------------------------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"eea64eddec12ea7a6221c8fd40bb79ac7f62074284efa8adfa6285e8d21d2b1a","failure_pattern":"ERROR CODE: InternalServerError; detail code AuthorizationPermissionMismatch; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["eea64eddec12ea7a6221c8fd40bb79ac7f62074284efa8adfa6285e8d21d2b1a"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4207/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026807165821390848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4207,"pr_state":"closed","pr_sha":"a02b279a61587d577a594a03ad7cf24d793edffe","final_merged_sha":"a02b279a61587d577a594a03ad7cf24d793edffe","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-25T23:50:59Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3780/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026807058761781248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3780,"pr_state":"closed","pr_sha":"dcf56c7037cbaf934aab3f6c5a1aa40173a34c36","final_merged_sha":"dcf56c7037cbaf934aab3f6c5a1aa40173a34c36","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-25T23:50:33Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"65222b5a04ee5712aa9e564bbfc959d9a9927e24e0f3ad8b914ea5b98f6ad072","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3780/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026807058761781248","occurred_at":"2026-02-25T23:50:33Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3956/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026807031557525504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3956,"pr_state":"closed","pr_sha":"66b441ca6c953e69bc787272a5e23c01c0d73f5b","final_merged_sha":"66b441ca6c953e69bc787272a5e23c01c0d73f5b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-25T23:50:27Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4218/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026804678104190976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4218,"pr_state":"closed","pr_sha":"438fac206c1907373a07a282d1ae272368182491","final_merged_sha":"7cbe20cfeaea24560965a517ee0964a9c659dbe3","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-25T23:41:06Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026797073608740864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4188,"pr_state":"closed","pr_sha":"9ff64c743d742d674acb9ecd209da84ed44f0ca5","final_merged_sha":"ad7003e5a941946bb5a1fed27c539b6ee3d95872","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-25T23:10:53Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4213/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026792596826755072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4213,"pr_state":"closed","pr_sha":"096bc31775b32d485c1da66c7c477745a134e33c","final_merged_sha":"096bc31775b32d485c1da66c7c477745a134e33c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-25T22:53:05Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026791785870659584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3679,"pr_state":"closed","pr_sha":"3ca804d0ed17f6519486da5d5d95366c60fc5e40","final_merged_sha":"3ca804d0ed17f6519486da5d5d95366c60fc5e40","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-25T22:49:52Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"13041cee146230312aeb0438ae8346115c26505b83001934f290f5135aff5c1c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026791785870659584","occurred_at":"2026-02-25T22:49:52Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:153]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:153]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"661c949af031ecc1c540435df4b95eed4a973f6dda2be7a26595e87a05bd3173","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026791785870659584","occurred_at":"2026-02-25T22:49:52Z","failed_at":"e2e","signature_id":"91efc5c10d5fa1c97df06fa6995f176ca7b3d8765353947120c174e2dad7ea4e","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000dd6360\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-bz5pm9\" to finish creating: GET https://rp.j0659584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9b308276-5a29-4bdc-9917-0a6f5eb47ab9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9b308276-5a29-4bdc-9917-0a6f5eb47ab9\",\n      \"name\": \"9b308276-5a29-4bdc-9917-0a6f5eb47ab9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-25T23:38:05.329111798Z\",\n      \"endTime\": \"2026-02-25T23:40:46.498452335Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"GET https://arohcpprowoidcj0659584.blob.core.windows.net/$web\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\nERROR CODE: AuthorizationPermissionMismatch\\n--------------------------------------------------------------------------------\\n\\u003c?xml version=\\\"1.0\\\" encoding=\\\"utf-8\\\"?\\u003e\\u003cError\\u003e\\u003cCode\\u003eAuthorizationPermissionMismatch\\u003c/Code\\u003e\\u003cMessage\\u003eThis request is not authorized to perform this operation using this permission.\\nRequestId:5b666743-501e-0081-38b0-a62840000000\\nTime:2026-02-25T23:40:12.7324517Z\\u003c/Message\\u003e\\u003c/Error\\u003e\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-bz5pm9\\\" to finish creating: GET https://rp.j0659584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9b308276-5a29-4bdc-9917-0a6f5eb47ab9\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9b308276-5a29-4bdc-9917-0a6f5eb47ab9\\\",\\n  \\\"name\\\": \\\"9b308276-5a29-4bdc-9917-0a6f5eb47ab9\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-25T23:38:05.329111798Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-25T23:40:46.498452335Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"GET https://arohcpprowoidcj0659584.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:5b666743-501e-0081-38b0-a62840000000\\\\nTime:2026-02-25T23:40:12.7324517Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000dd6340\u003e{\n            msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-bz5pm9\\\" to finish creating: GET https://rp.j0659584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9b308276-5a29-4bdc-9917-0a6f5eb47ab9\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9b308276-5a29-4bdc-9917-0a6f5eb47ab9\\\",\\n  \\\"name\\\": \\\"9b308276-5a29-4bdc-9917-0a6f5eb47ab9\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-25T23:38:05.329111798Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-25T23:40:46.498452335Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"GET https://arohcpprowoidcj0659584.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:5b666743-501e-0081-38b0-a62840000000\\\\nTime:2026-02-25T23:40:12.7324517Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n---------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000dd6360\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-bz5pm9\" to finish creating: GET https://rp.j0659584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9b308276-5a29-4bdc-9917-0a6f5eb47ab9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9b308276-5a29-4bdc-9917-0a6f5eb47ab9\",\n      \"name\": \"9b308276-5a29-4bdc-9917-0a6f5eb47ab9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-02-25T23:38:05.329111798Z\",\n      \"endTime\": \"2026-02-25T23:40:46.498452335Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"GET https://arohcpprowoidcj0659584.blob.core.windows.net/$web\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\nERROR CODE: AuthorizationPermissionMismatch\\n--------------------------------------------------------------------------------\\n\\u003c?xml version=\\\"1.0\\\" encoding=\\\"utf-8\\\"?\\u003e\\u003cError\\u003e\\u003cCode\\u003eAuthorizationPermissionMismatch\\u003c/Code\\u003e\\u003cMessage\\u003eThis request is not authorized to perform this operation using this permission.\\nRequestId:5b666743-501e-0081-38b0-a62840000000\\nTime:2026-02-25T23:40:12.7324517Z\\u003c/Message\\u003e\\u003c/Error\\u003e\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-bz5pm9\\\" to finish creating: GET https://rp.j0659584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9b308276-5a29-4bdc-9917-0a6f5eb47ab9\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9b308276-5a29-4bdc-9917-0a6f5eb47ab9\\\",\\n  \\\"name\\\": \\\"9b308276-5a29-4bdc-9917-0a6f5eb47ab9\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-25T23:38:05.329111798Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-25T23:40:46.498452335Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"GET https://arohcpprowoidcj0659584.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:5b666743-501e-0081-38b0-a62840000000\\\\nTime:2026-02-25T23:40:12.7324517Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000dd6340\u003e{\n            msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-bz5pm9\\\" to finish creating: GET https://rp.j0659584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9b308276-5a29-4bdc-9917-0a6f5eb47ab9\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9b308276-5a29-4bdc-9917-0a6f5eb47ab9\\\",\\n  \\\"name\\\": \\\"9b308276-5a29-4bdc-9917-0a6f5eb47ab9\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-02-25T23:38:05.329111798Z\\\",\\n  \\\"endTime\\\": \\\"2026-02-25T23:40:46.498452335Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"GET https://arohcpprowoidcj0659584.blob.core.windows.net/$web\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 This request is not authorized to perform this operation using this permission.\\\\nERROR CODE: AuthorizationPermissionMismatch\\\\n--------------------------------------------------------------------------------\\\\n\\ufeff\\\\u003c?xml version=\\\\\\\"1.0\\\\\\\" encoding=\\\\\\\"utf-8\\\\\\\"?\\\\u003e\\\\u003cError\\\\u003e\\\\u003cCode\\\\u003eAuthorizationPermissionMismatch\\\\u003c/Code\\\\u003e\\\\u003cMessage\\\\u003eThis request is not authorized to perform this operation using this permission.\\\\nRequestId:5b666743-501e-0081-38b0-a62840000000\\\\nTime:2026-02-25T23:40:12.7324517Z\\\\u003c/Message\\\\u003e\\\\u003c/Error\\\\u003e\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n---------------------------------...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"eea64eddec12ea7a6221c8fd40bb79ac7f62074284efa8adfa6285e8d21d2b1a","failure_pattern":"ERROR CODE: InternalServerError; detail code AuthorizationPermissionMismatch; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","eea64eddec12ea7a6221c8fd40bb79ac7f62074284efa8adfa6285e8d21d2b1a"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4208/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026784334505054208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4208,"pr_state":"closed","pr_sha":"c8160d3983282ee9fdf3e2f47f59530a011c9e1c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T22:20:15Z"},"failed_at":["e2e"],"failed_test_count":15,"occurrences":[{"row_id":"0d79dd0c96fe6a6565d763826d72f7912fec903e53f66d3a1948dd84710f1cef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4208/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026784334505054208","occurred_at":"2026-02-25T22:20:15Z","failed_at":"e2e","signature_id":"eff21110f0cf90869043e17d3f1955d3c621d10837a18fe32042e22f9060ae08","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f783f0\u003e: \n    failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-nb4hmc, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-nb4hmc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-nb4hmc, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-nb4hmc, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-nb4hmc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-nb4hmc, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000d1dae0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-nb4hmc\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000f783c0\u003e{\n                msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-nb4hmc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-nb4hmc, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000d1dae0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-nb4hmc\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f783f0\u003e: \n    failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-nb4hmc, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-nb4hmc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-nb4hmc, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-nb4hmc, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-nb4hmc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-nb4hmc, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000d1dae0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-nb4hmc\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000f783c0\u003e{\n                msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-nb4hmc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-nb4hmc, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000d1dae0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-nb4hmc\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qtsmtv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: context deadline exceeded"}},{"row_id":"10a22e9fb28d168d9b72c3fef158c6091d9d33b9b96b379facb67a94bae10f5d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4208/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026784334505054208","occurred_at":"2026-02-25T22:20:15Z","failed_at":"e2e","signature_id":"8319206020c6f53f48a73125d647ff86c4b02ec615c6ca60fcfd5a539518fde1","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"\u003e github.com/Azure/ARO-HCP/test/e2e.init.func15.2({0x7f08e4594240, 0xc0009c44e0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n                | \n                | By(\"waiting for first cluster to complete creation\")\n                \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n                | \tFrequency: framework.StandardPollInterval,\n                | })\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8076598?, 0xc0009c44e0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 111\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 02/26/26 00:45:19.484\n  \"ts\"=\"2026-02-26 00:45:19.484354\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-02-26 00:45:19.484434\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-vc4bkz\" \"resourceGroup\"=\"customer-rg-lkw59k\"\n  \"ts\"=\"2026-02-26 00:45:20.125570\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-vc4bkz\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-02-26 00:45:20.656362\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-khzg2s\" \"resourceGroup\"=\"customer-rg-lkw59k\"\n  \"ts\"=\"2026-02-26 00:45:20.813594\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-khzg2s\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-02-26 00:45:21.315441\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-02-26 00:45:21.315574\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-02-26 00:45:21.315650\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-02-26 00:45:21.316740\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-165\"\n  \"ts\"=\"2026-02-26 00:45:21.316755\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-211\"\n  \"ts\"=\"2026-02-26 00:45:21.316780\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-lkw59k\"\n  \"ts\"=\"2026-02-26 00:45:21.316816\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-112\"\n  \"ts\"=\"2026-02-26 00:45:21.945252\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-165\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-26 00:45:22.307425\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-211\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-26 00:45:22.837155\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-lkw59k\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n  \"ts\"=\"2026-02-26 00:45:23.153215\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-112\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-26 00:45:23.589385\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-165\" \"deployment\"=\"msi-deployment-165\"\n  \"ts\"=\"2026-02-26 00:45:23.950013\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-211\" \"deployment\"=\"msi-deployment-211\"\n  \"ts\"=\"2026-02-26 00:45:24.408245\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-lkw59k\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-khzg2s\"\n  \"ts\"=\"2026-02-26 00:45:24.819953\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-112\" \"deployment\"=\"msi-deployment-112\"\n  \"ts\"=\"2026-02-26 00:45:26.350566\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-lkw59k\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-02-26 00:45:26.992581\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-lkw59k\" \"deployment\"=\"customer-infra-basic-hcp-cluster-vc4bkz\"\n  \"ts\"=\"2026-02-26 00:45:27.598895\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-02-26 00:45:27.599053\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-02-26 00:45:27.599173\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-lkw59k\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n\u003e github.com/Azure/ARO-HCP/test/e2e.init.func15.2({0x7f08e4594240, 0xc0009c44e0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n                | \n                | By(\"waiting for first cluster to complete creation\")\n                \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n                | \tFrequency: framework.StandardPollInterval,\n                | })\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8076598?, 0xc0009c44e0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 111\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 02/26/26 00:45:19.484\n  \"ts\"=\"2026-02-26 00:45:19.484354\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-02-26 00:45:19.484434\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-vc4bkz\" \"resourceGroup\"=\"customer-rg-lkw59k\"\n  \"ts\"=\"2026-02-26 00:45:20.125570\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-vc4bkz\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-02-26 00:45:20.656362\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-khzg2s\" \"resourceGroup\"=\"customer-rg-lkw59k\"\n  \"ts\"=\"2026-02-26 00:45:20.813594\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-khzg2s\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-02-26 00:45:21.315441\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-02-26 00:45:21.315574\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-02-26 00:45:21.315650\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-02-26 00:45:21.316740\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-165\"\n  \"ts\"=\"2026-02-26 00:45:21.316755\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-211\"\n  \"ts\"=\"2026-02-26 00:45:21.316780\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-lkw59k\"\n  \"ts\"=\"2026-02-26 00:45:21.316816\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-112\"\n  \"ts\"=\"2026-02-26 00:45:21.945252\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-165\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-26 00:45:22.307425\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-211\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-26 00:45:22.837155\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-lkw59k\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n  \"ts\"=\"2026-02-26 00:45:23.153215\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-112\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-26 00:45:23.589385\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-165\" \"deployment\"=\"msi-deployment-165\"\n  \"ts\"=\"2026-02-26 00:45:23.950013\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-211\" \"deployment\"=\"msi-deployment-211\"\n  \"ts\"=\"2026-02-26 00:45:24.408245\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-lkw59k\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-khzg2s\"\n  \"ts\"=\"2026-02-26 00:45:24.819953\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-dev-112\" \"deployment\"=\"msi-deployment-112\"\n  \"ts\"=\"2026-02-26 00:45:26.350566\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-lkw59k\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-02-26 00:45:26.992581\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-lkw59k\" \"deployment\"=\"customer-infra-basic-hcp-cluster-vc4bkz\"\n  \"ts\"=\"2026-02-26 00:45:27.598895\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-02-26 00:45:27.599053\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-02-26 00:45:27.599173\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-lkw59k\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"10d932a30f8d844f4722f545cdbecce008b6e6af29fd3cf97ecca57055710b85","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4208/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026784334505054208","occurred_at":"2026-02-25T22:20:15Z","failed_at":"e2e","signature_id":"232cce03d89d60f963f15f1746c65adeefca44ab0bd828314ebcc3a7946e44e2","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010ec690\u003e: \n    failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-q5thkp, error: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-q5thkp\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-q5thkp, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-q5thkp, error: failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-q5thkp\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-q5thkp, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0010af610\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-q5thkp\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0010ec660\u003e{\n                msg: \"failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-q5thkp\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-q5thkp, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0010af610\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-q5thkp\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010ec690\u003e: \n    failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-q5thkp, error: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-q5thkp\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-q5thkp, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster basic-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-q5thkp, error: failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-q5thkp\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-q5thkp, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0010af610\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-q5thkp\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0010ec660\u003e{\n                msg: \"failed waiting for cluster=\\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-q5thkp\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-q5thkp, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0010af610\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-q5thkp\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qtsmtv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: context deadline exceeded"}},{"row_id":"191fd4d1274a0cc479fbe1c48a83bb97c3f2b3184a1cc30a6e31bed206ee6417","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4208/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026784334505054208","occurred_at":"2026-02-25T22:20:15Z","failed_at":"e2e","signature_id":"a2867aaf785b4f712c9c92b63d898ee144b0077d4cd0b5e50024a6b3ba18047f","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00155b3e0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7dwxjz, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-7dwxjz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7dwxjz, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7dwxjz, error: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-7dwxjz\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7dwxjz, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000dadd40\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7dwxjz\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00155b3b0\u003e{\n                msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-7dwxjz\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7dwxjz, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000dadd40\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7dwxjz\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00155b3e0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7dwxjz, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-7dwxjz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7dwxjz, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7dwxjz, error: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-7dwxjz\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7dwxjz, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000dadd40\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7dwxjz\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00155b3b0\u003e{\n                msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-7dwxjz\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7dwxjz, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000dadd40\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7dwxjz\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qtsmtv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: context deadline exceeded"}},{"row_id":"2bb8d20a930c39cf2977d09bafc997834d1f35ec1bafe52e19a3cb758f6363a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4208/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026784334505054208","occurred_at":"2026-02-25T22:20:15Z","failed_at":"e2e","signature_id":"12f31092e4626f259300ddf220cc4e638a8edb24f79758025e32b8682414b848","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001167770\u003e: \n    failed to create HCP cluster list-test-cluster-2z4qgp, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-2z4qgp in resource group cluster-listing-qttmzq, error: failed waiting for cluster=\"list-test-cluster-2z4qgp\" in resourcegroup=\"cluster-listing-qttmzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-2z4qgp in resource group cluster-listing-qttmzq, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster list-test-cluster-2z4qgp, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-2z4qgp in resource group cluster-listing-qttmzq, error: failed waiting for cluster=\\\"list-test-cluster-2z4qgp\\\" in resourcegroup=\\\"cluster-listing-qttmzq\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-2z4qgp in resource group cluster-listing-qttmzq, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001150800\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-2z4qgp in resource group cluster-listing-qttmzq\",\n            },\n            \u003c*fmt.wrapErrors | 0xc001167740\u003e{\n                msg: \"failed waiting for cluster=\\\"list-test-cluster-2z4qgp\\\" in resourcegroup=\\\"cluster-listing-qttmzq\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-2z4qgp in resource group cluster-listing-qttmzq, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001150800\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-2z4qgp in resource group cluster-listing-qttmzq\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001167770\u003e: \n    failed to create HCP cluster list-test-cluster-2z4qgp, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-2z4qgp in resource group cluster-listing-qttmzq, error: failed waiting for cluster=\"list-test-cluster-2z4qgp\" in resourcegroup=\"cluster-listing-qttmzq\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-2z4qgp in resource group cluster-listing-qttmzq, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster list-test-cluster-2z4qgp, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-2z4qgp in resource group cluster-listing-qttmzq, error: failed waiting for cluster=\\\"list-test-cluster-2z4qgp\\\" in resourcegroup=\\\"cluster-listing-qttmzq\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-2z4qgp in resource group cluster-listing-qttmzq, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001150800\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-2z4qgp in resource group cluster-listing-qttmzq\",\n            },\n            \u003c*fmt.wrapErrors | 0xc001167740\u003e{\n                msg: \"failed waiting for cluster=\\\"list-test-cluster-2z4qgp\\\" in resourcegroup=\\\"cluster-listing-qttmzq\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-2z4qgp in resource group cluster-listing-qttmzq, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001150800\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-2z4qgp in resource group cluster-listing-qttmzq\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qtsmtv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: context deadline exceeded"}},{"row_id":"2e5ee278234337233a65dd8e3942a818b3d74976b329ee1cbc73581dbb7adb68","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4208/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026784334505054208","occurred_at":"2026-02-25T22:20:15Z","failed_at":"e2e","signature_id":"b5071b45276cdd7205b7c1ef2ec35b3e7c2193e3c81df66a9c8813076c7d3f87","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000fd8870\u003e: \n    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-jg87w4, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-jg87w4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-jg87w4, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-jg87w4, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-jg87w4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-jg87w4, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000975040\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-jg87w4\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000fd8840\u003e{\n                msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-jg87w4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-jg87w4, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000975040\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-jg87w4\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000fd8870\u003e: \n    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-jg87w4, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-jg87w4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-jg87w4, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-jg87w4, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-jg87w4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-jg87w4, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000975040\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-jg87w4\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000fd8840\u003e{\n                msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-jg87w4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-jg87w4, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000975040\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-jg87w4\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qtsmtv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: context deadline exceeded"}},{"row_id":"4a97bcb00ba5bc4b25a96ee09dd1f0f2ddef15cb4f111a1990ba1e1aac782c64","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4208/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026784334505054208","occurred_at":"2026-02-25T22:20:15Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"4f6f98d50ddbfd71958ba3d2f05b1d9f4252f47952db2a0c257764e6dabc8fb3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4208/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026784334505054208","occurred_at":"2026-02-25T22:20:15Z","failed_at":"e2e","signature_id":"0d26a30b450bd559430a51c9858cf38b9e132a6fc895768f6e081b6071906ae1","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000c44c60\u003e: \n    failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-dxmbbp, error: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-dxmbbp\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-dxmbbp, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-dxmbbp, error: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-dxmbbp\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-dxmbbp, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0006a0fa0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-dxmbbp\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000c44c30\u003e{\n                msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-dxmbbp\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-dxmbbp, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0006a0fa0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-dxmbbp\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000c44c60\u003e: \n    failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-dxmbbp, error: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-dxmbbp\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-dxmbbp, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-dxmbbp, error: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-dxmbbp\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-dxmbbp, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0006a0fa0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-dxmbbp\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000c44c30\u003e{\n                msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-dxmbbp\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-dxmbbp, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0006a0fa0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-dxmbbp\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qtsmtv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: context deadline exceeded"}},{"row_id":"7e4444dbed7030d067cd7dcdd8d247e460e0f0f983e0c7d5114c49a6ff260c3f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4208/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026784334505054208","occurred_at":"2026-02-25T22:20:15Z","failed_at":"e2e","signature_id":"b8285d1fe5e0e06b0118077c74385230269c41bde5a189b9513acacc0d1a338e","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bc75cb8f745e5837337ba0536c5a0bca78a86948c3d2f51166b94f8c6fca3f5c","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes"}},{"row_id":"8c2a475da331ebfedde13103cfc9061cf4d74822b44f242c8edb8589ec9a6c0c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4208/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026784334505054208","occurred_at":"2026-02-25T22:20:15Z","failed_at":"e2e","signature_id":"928280fc77042ccb893af7aaab4cd1f180b37b84cf06e87551ebdd6dda74d760","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011d0d80\u003e: \n    failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-bwmbmd, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-bwmbmd\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-bwmbmd, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-bwmbmd, error: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-bwmbmd\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-bwmbmd, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001309a10\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-bwmbmd\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0011d0d50\u003e{\n                msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-bwmbmd\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-bwmbmd, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001309a10\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-bwmbmd\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011d0d80\u003e: \n    failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-bwmbmd, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-bwmbmd\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-bwmbmd, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-bwmbmd, error: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-bwmbmd\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-bwmbmd, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001309a10\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-bwmbmd\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0011d0d50\u003e{\n                msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-bwmbmd\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-bwmbmd, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001309a10\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-bwmbmd\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qtsmtv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: context deadline exceeded"}},{"row_id":"ac1b27ce3c498092090cece579500a5754720f9352c5f4906865d780b7fd11cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4208/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026784334505054208","occurred_at":"2026-02-25T22:20:15Z","failed_at":"e2e","signature_id":"8aa61fc94bfe81a5eda99273c78879fc7fbddafc798cc60f9d5670d1b2e693f6","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a4e420\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qtsmtv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-qtsmtv\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0008bc0e0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000a4e3f0\u003e{\n                msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-qtsmtv\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0008bc0e0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a4e420\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qtsmtv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-qtsmtv\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0008bc0e0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000a4e3f0\u003e{\n                msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-qtsmtv\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0008bc0e0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qtsmtv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: context deadline exceeded"}},{"row_id":"af5728d3f79e681e66964bf0c2c938dc31422ca2531cafb2cae5535d632ce376","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4208/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026784334505054208","occurred_at":"2026-02-25T22:20:15Z","failed_at":"e2e","signature_id":"18584dec2fbf7633aef007d2de83e840608e478b1b66b05bda0d0f32650743fb","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ecd5f0\u003e: \n    failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pzm49x, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-pzm49x\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pzm49x, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pzm49x, error: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-pzm49x\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pzm49x, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00117e010\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pzm49x\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000ecd5c0\u003e{\n                msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-pzm49x\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pzm49x, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc00117e010\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pzm49x\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ecd5f0\u003e: \n    failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pzm49x, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-pzm49x\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pzm49x, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pzm49x, error: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-pzm49x\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pzm49x, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00117e010\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pzm49x\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000ecd5c0\u003e{\n                msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-pzm49x\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pzm49x, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc00117e010\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-pzm49x\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qtsmtv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: context deadline exceeded"}},{"row_id":"b3b2bc3630e10bb9b258a07a26b39e2931dfc44a0a8b0b799c46862f07ef3d8c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4208/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026784334505054208","occurred_at":"2026-02-25T22:20:15Z","failed_at":"e2e","signature_id":"06cfbfba4944fa7e100c2ae19fecd6c0ceeebb8d7e6dc83a5bdd7b9c87143d0b","test_name":"Customer should be able to create an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015fc6c0\u003e: \n    failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-5kgdld, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-5kgdld\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-5kgdld, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-5kgdld, error: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-5kgdld\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-5kgdld, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001720cd0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-5kgdld\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0015fc690\u003e{\n                msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-5kgdld\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-5kgdld, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001720cd0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-5kgdld\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015fc6c0\u003e: \n    failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-5kgdld, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-5kgdld\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-5kgdld, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-5kgdld, error: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-5kgdld\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-5kgdld, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001720cd0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-5kgdld\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0015fc690\u003e{\n                msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-5kgdld\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-5kgdld, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001720cd0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-5kgdld\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qtsmtv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: context deadline exceeded"}},{"row_id":"c14c2e1636498d21ff8d89a6fcf88e2b11363cbf3860483df827b9259eb74246","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4208/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026784334505054208","occurred_at":"2026-02-25T22:20:15Z","failed_at":"e2e","signature_id":"1e3404208d8b54765ff0192ba2f7de24982c82c50f6cef8ce030496e75662050","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:100]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011669f0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-b9dtbr, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-b9dtbr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-b9dtbr, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-b9dtbr, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-b9dtbr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-b9dtbr, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00115e360\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-b9dtbr\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0011669c0\u003e{\n                msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-b9dtbr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-b9dtbr, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc00115e360\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-b9dtbr\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:100]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011669f0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-b9dtbr, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-b9dtbr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-b9dtbr, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-b9dtbr, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-b9dtbr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-b9dtbr, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00115e360\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-b9dtbr\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0011669c0\u003e{\n                msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-b9dtbr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-b9dtbr, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc00115e360\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-b9dtbr\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qtsmtv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: context deadline exceeded"}},{"row_id":"f5639b497bc78b8f693faf5c79e8f00eff38949c88ffb98b1371e37365f0df2d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4208/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026784334505054208","occurred_at":"2026-02-25T22:20:15Z","failed_at":"e2e","signature_id":"e5d77e709d317153218caa355d9228ee0b88e0adfc32ad95ced64299537ac0ce","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d5b350\u003e: \n    failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-b6zr97, error: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-b6zr97\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-b6zr97, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-b6zr97, error: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-b6zr97\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-b6zr97, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000f67b00\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-b6zr97\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000d5b320\u003e{\n                msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-b6zr97\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-b6zr97, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000f67b00\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-b6zr97\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:87]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d5b350\u003e: \n    failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-b6zr97, error: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-b6zr97\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-b6zr97, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster engineering-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-b6zr97, error: failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-b6zr97\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-b6zr97, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000f67b00\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-b6zr97\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000d5b320\u003e{\n                msg: \"failed waiting for cluster=\\\"engineering-hcp-cluster\\\" in resourcegroup=\\\"kusto-logs-b6zr97\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-b6zr97, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000f67b00\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster engineering-hcp-cluster in resource group kusto-logs-b6zr97\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qtsmtv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-qtsmtv, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":15,"failure_pattern_ids":["75fe4fe524930ef30df14def2711e3f1d7e0ed489b7726da52684d3d8d655bf4","9beb4edaf388b0da56e666b45689d676c4c9672098363e2e0b884ed6e85ffb18","bc75cb8f745e5837337ba0536c5a0bca78a86948c3d2f51166b94f8c6fca3f5c","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":15,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026783597234491392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3679,"pr_state":"closed","pr_sha":"3ca804d0ed17f6519486da5d5d95366c60fc5e40","final_merged_sha":"3ca804d0ed17f6519486da5d5d95366c60fc5e40","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-25T22:17:19Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d464f686499df6eb47d00ca578769237bdd32515dddbb6c74e2b72a68f602e9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026783597234491392","occurred_at":"2026-02-25T22:17:19Z","failed_at":"provision","signature_id":"12e4c71a9a2f27c3933827b33d6567a5995c280323c72629a422bc5c75fcc5f8","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-azure-plugin-image","test_suite":"step graph","failure_text":"time=2026-02-25T22:37:27.977Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-azure-plugin-image description=\"Step mirror-azure-plugin-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e velero.azurePlugin.registry \u003cnil\u003e}:{\u003cnil\u003e velero.azurePlugin.repository \u003cnil\u003e}@{\u003cnil\u003e velero.azurePlugin.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T22:37:30.343Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-azure-plugin-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.redhat.io/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9@sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e to arohcpsvcdev.azurecr.io/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e.\\nThe image will still be available under it's original digest sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e in the target registry.\\nError response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223","failure_pattern":"Error response from registry: failed to resolve sha256:\u003chex\u003e: GET \"\u003curl\u003e\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: \u003curl\u003e","search_query":"Error response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4212/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026779874118602752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4212,"pr_state":"closed","pr_sha":"551e259a36be7a8b6e5199c23a5f2890117fae0f","final_merged_sha":"551e259a36be7a8b6e5199c23a5f2890117fae0f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-25T22:02:32Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5c818c530938486dea3e4d81deb749a8ddf225ea5dfcd969c230e822d3b8684a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4212/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026779874118602752","occurred_at":"2026-02-25T22:02:32Z","failed_at":"provision","signature_id":"dc988d340dad9de51c7b30f776b40dac5ac8a9e7fddd9a4a2a84de672db2304c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-hypershift-plugin-image","test_suite":"step graph","failure_text":"time=2026-02-25T22:44:52.734Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-hypershift-plugin-image description=\"Step mirror-hypershift-plugin-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e velero.hypershiftPlugin.registry \u003cnil\u003e}:{\u003cnil\u003e velero.hypershiftPlugin.repository \u003cnil\u003e}@{\u003cnil\u003e velero.hypershiftPlugin.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T22:44:55.660Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-hypershift-plugin-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.redhat.io/oadp/oadp-hypershift-velero-plugin-rhel9@sha256:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e to arohcpsvcdev.azurecr.io/oadp/oadp-hypershift-velero-plugin-rhel9:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e.\\nThe image will still be available under it's original digest sha256:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e in the target registry.\\nError response from registry: failed to resolve sha256:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-hypershift-velero-plugin-rhel9/manifests/sha256:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223","failure_pattern":"Error response from registry: failed to resolve sha256:\u003chex\u003e: GET \"\u003curl\u003e\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: \u003curl\u003e","search_query":"Error response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4216/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026775005714452480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4216,"pr_state":"closed","pr_sha":"e9d3aedf019f7de004555a103ff4e2680708099e","final_merged_sha":"e9d3aedf019f7de004555a103ff4e2680708099e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-25T21:43:11Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"dfaa7eb102bc1853b1cf838a18a4956bd2050c79593142c881bafa549e8140be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4216/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026775005714452480","occurred_at":"2026-02-25T21:43:11Z","failed_at":"provision","signature_id":"dc988d340dad9de51c7b30f776b40dac5ac8a9e7fddd9a4a2a84de672db2304c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-hypershift-plugin-image","test_suite":"step graph","failure_text":"time=2026-02-25T22:27:52.583Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-hypershift-plugin-image description=\"Step mirror-hypershift-plugin-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e velero.hypershiftPlugin.registry \u003cnil\u003e}:{\u003cnil\u003e velero.hypershiftPlugin.repository \u003cnil\u003e}@{\u003cnil\u003e velero.hypershiftPlugin.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T22:27:55.317Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-hypershift-plugin-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.redhat.io/oadp/oadp-hypershift-velero-plugin-rhel9@sha256:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e to arohcpsvcdev.azurecr.io/oadp/oadp-hypershift-velero-plugin-rhel9:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e.\\nThe image will still be available under it's original digest sha256:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e in the target registry.\\nError response from registry: failed to resolve sha256:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-hypershift-velero-plugin-rhel9/manifests/sha256:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223","failure_pattern":"Error response from registry: failed to resolve sha256:\u003chex\u003e: GET \"\u003curl\u003e\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: \u003curl\u003e","search_query":"Error response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026770808747069440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3679,"pr_state":"closed","pr_sha":"3ca804d0ed17f6519486da5d5d95366c60fc5e40","final_merged_sha":"3ca804d0ed17f6519486da5d5d95366c60fc5e40","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-25T21:19:14Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"2028e8411ff6e8425ae61f4348a28545cfd22efe55512e829f66627ae59702b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026770808747069440","occurred_at":"2026-02-25T21:19:14Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-02-25T21:36:37.127Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-02-25T21:36:39.407Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-02-25T21:36:40.831Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=989935f8672b1a8791202138e4b994003547af7139d0aa916f5d7417cf6d6f38\ntime=2026-02-25T21:38:11.597Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7069440/providers/Microsoft.Resources/deployments/989935f8672b1a8791202138e4b994003547af7139d0aa916f5d7417cf6d6f38/operationStatuses/08584295514854648601\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7069440/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7069440/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j7069440\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 0ca16638-1521-4555-b306-e0b991d316b6:2/25/2026 9:37:57 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3780/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026758580694159360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3780,"pr_state":"closed","pr_sha":"dcf56c7037cbaf934aab3f6c5a1aa40173a34c36","final_merged_sha":"dcf56c7037cbaf934aab3f6c5a1aa40173a34c36","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-25T20:37:55Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c0165539831f5a6fdca8d8ac756b97d930e5c28e13ec2a30408408463bc86ae6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3780/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026758580694159360","occurred_at":"2026-02-25T20:37:55Z","failed_at":"provision","signature_id":"12e4c71a9a2f27c3933827b33d6567a5995c280323c72629a422bc5c75fcc5f8","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-azure-plugin-image","test_suite":"step graph","failure_text":"time=2026-02-25T21:29:36.893Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-azure-plugin-image description=\"Step mirror-azure-plugin-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e velero.azurePlugin.registry \u003cnil\u003e}:{\u003cnil\u003e velero.azurePlugin.repository \u003cnil\u003e}@{\u003cnil\u003e velero.azurePlugin.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T21:29:39.584Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-azure-plugin-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.redhat.io/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9@sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e to arohcpsvcdev.azurecr.io/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e.\\nThe image will still be available under it's original digest sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e in the target registry.\\nError response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223","failure_pattern":"Error response from registry: failed to resolve sha256:\u003chex\u003e: GET \"\u003curl\u003e\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: \u003curl\u003e","search_query":"Error response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4208/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026757955185020928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4208,"pr_state":"closed","pr_sha":"c8160d3983282ee9fdf3e2f47f59530a011c9e1c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T20:35:26Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3aa6a03d3e589fde5289b02b6b983b3a003e9e49d6248aa9928c2ab5803dcdb7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4208/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026757955185020928","occurred_at":"2026-02-25T20:35:26Z","failed_at":"provision","signature_id":"dc988d340dad9de51c7b30f776b40dac5ac8a9e7fddd9a4a2a84de672db2304c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-hypershift-plugin-image","test_suite":"step graph","failure_text":"time=2026-02-25T21:20:24.336Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-hypershift-plugin-image description=\"Step mirror-hypershift-plugin-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e velero.hypershiftPlugin.registry \u003cnil\u003e}:{\u003cnil\u003e velero.hypershiftPlugin.repository \u003cnil\u003e}@{\u003cnil\u003e velero.hypershiftPlugin.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T21:20:26.980Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-hypershift-plugin-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.redhat.io/oadp/oadp-hypershift-velero-plugin-rhel9@sha256:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e to arohcpsvcdev.azurecr.io/oadp/oadp-hypershift-velero-plugin-rhel9:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e.\\nThe image will still be available under it's original digest sha256:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e in the target registry.\\nError response from registry: failed to resolve sha256:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-hypershift-velero-plugin-rhel9/manifests/sha256:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223","failure_pattern":"Error response from registry: failed to resolve sha256:\u003chex\u003e: GET \"\u003curl\u003e\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: \u003curl\u003e","search_query":"Error response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4213/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026755061874429952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4213,"pr_state":"closed","pr_sha":"096bc31775b32d485c1da66c7c477745a134e33c","final_merged_sha":"096bc31775b32d485c1da66c7c477745a134e33c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-25T20:23:56Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d34a2e191a2cab78d97643c62c99634602803aed5135813d5e0511f0185dfdd7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4213/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026755061874429952","occurred_at":"2026-02-25T20:23:56Z","failed_at":"provision","signature_id":"12e4c71a9a2f27c3933827b33d6567a5995c280323c72629a422bc5c75fcc5f8","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-azure-plugin-image","test_suite":"step graph","failure_text":"time=2026-02-25T21:12:48.575Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-azure-plugin-image description=\"Step mirror-azure-plugin-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e velero.azurePlugin.registry \u003cnil\u003e}:{\u003cnil\u003e velero.azurePlugin.repository \u003cnil\u003e}@{\u003cnil\u003e velero.azurePlugin.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T21:12:50.788Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-azure-plugin-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.redhat.io/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9@sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e to arohcpsvcdev.azurecr.io/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e.\\nThe image will still be available under it's original digest sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e in the target registry.\\nError response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223","failure_pattern":"Error response from registry: failed to resolve sha256:\u003chex\u003e: GET \"\u003curl\u003e\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: \u003curl\u003e","search_query":"Error response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4211/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026751698529161216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4211,"pr_state":"open","pr_sha":"8e248ec000e5abcd96273d899a30516dd1e769b8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T20:10:34Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3c9638c4b4d8a0bad7c7109c38987c66a510f2f6b39ab3a30e3f77e651ee091a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4211/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026751698529161216","occurred_at":"2026-02-25T20:10:34Z","failed_at":"provision","signature_id":"12e4c71a9a2f27c3933827b33d6567a5995c280323c72629a422bc5c75fcc5f8","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-azure-plugin-image","test_suite":"step graph","failure_text":"time=2026-02-25T20:58:33.207Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-azure-plugin-image description=\"Step mirror-azure-plugin-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e velero.azurePlugin.registry \u003cnil\u003e}:{\u003cnil\u003e velero.azurePlugin.repository \u003cnil\u003e}@{\u003cnil\u003e velero.azurePlugin.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T20:58:35.555Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-azure-plugin-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.redhat.io/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9@sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e to arohcpsvcdev.azurecr.io/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e.\\nThe image will still be available under it's original digest sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e in the target registry.\\nError response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223","failure_pattern":"Error response from registry: failed to resolve sha256:\u003chex\u003e: GET \"\u003curl\u003e\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: \u003curl\u003e","search_query":"Error response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026748626184704000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3679,"pr_state":"closed","pr_sha":"3ca804d0ed17f6519486da5d5d95366c60fc5e40","final_merged_sha":"3ca804d0ed17f6519486da5d5d95366c60fc5e40","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-25T19:58:22Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ebd503cba85f3c5bdce93220675424a58e463f129371aacc3dc36c9284f97f4f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026748626184704000","occurred_at":"2026-02-25T19:58:22Z","failed_at":"provision","signature_id":"8502a51bc863354d73982a77213058401ba8e85e4e534c9c54c944ad29110fef","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-velero-image","test_suite":"step graph","failure_text":"time=2026-02-25T20:59:12.201Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image description=\"Step mirror-velero-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e velero.server.registry \u003cnil\u003e}:{\u003cnil\u003e velero.server.repository \u003cnil\u003e}@{\u003cnil\u003e velero.server.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T20:59:14.874Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.redhat.io/oadp/oadp-velero-rhel9@sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b to arohcpsvcdev.azurecr.io/oadp/oadp-velero-rhel9:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b.\\nThe image will still be available under it's original digest sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b in the target registry.\\nError response from registry: failed to resolve sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-rhel9/manifests/sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223","failure_pattern":"Error response from registry: failed to resolve sha256:\u003chex\u003e: GET \"\u003curl\u003e\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: \u003curl\u003e","search_query":"Error response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4212/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026747813072736256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4212,"pr_state":"closed","pr_sha":"395fd8316ddbcee82632b1ac8c16734e64825f81","final_merged_sha":"551e259a36be7a8b6e5199c23a5f2890117fae0f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T19:39:46Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"4d7566650f8f8e91fa434b91aee2c1699f6ebba9e66e7c2b20aacd184faaab72","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4212/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026747813072736256","occurred_at":"2026-02-25T19:39:46Z","failed_at":"provision","signature_id":"c027c4d12f6a3d02006d0282f3d5b1020328ed77f7bf43d76c7a5e7566e656ac","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"step graph","failure_text":"time=2026-02-25T20:35:16.832Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-02-25T20:35:18.519Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-02-25T20:35:20.565Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=26b9c29583a7924ad3a98fccd321c157eec0f1139ce2debf0027ac3e6c51faca\ntime=2026-02-25T20:50:24.199Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j2736256-svc/providers/Microsoft.Resources/deployments/26b9c29583a7924ad3a98fccd321c157eec0f1139ce2debf0027ac3e6c51faca/operationStatuses/08584295551659603455\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j2736256-svc/providers/Microsoft.Resources/deployments/maestro-server\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/maestro-eg-crt-rcvcdydzmzklu\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n                \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deployments/maestro-eg-crt-rcvcdydzmzklu\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n                \\\\\\\"details\\\\\\\": [\\\\r\\\\n                  {\\\\r\\\\n                    \\\\\\\"code\\\\\\\": \\\\\\\"InvalidTemplateDeployment\\\\\\\",\\\\r\\\\n                    \\\\\\\"message\\\\\\\": \\\\\\\"The template deployment 'maestro-server-j2736256-cert' is not valid according to the validation procedure. The tracking id is '437861dd-327e-4979-892c-30ec68200dc1'. See inner errors for details.\\\\\\\",\\\\r\\\\n                    \\\\\\\"details\\\\\\\": [\\\\r\\\\n                      {\\\\r\\\\n                        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentPreflightValidationFailed\\\\\\\",\\\\r\\\\n                        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deploymentScripts/newCertwithRotationKV-maestro-server-j2736256\\\\\\\",\\\\r\\\\n                        \\\\\\\"message\\\\\\\": \\\\\\\"The script language version '15.3.0' is not supported. Supported versions: '2.7,2.8,3.0,3.1,3.2,3.3,3.4,3.5,3.6,3.7,3.8,4.1,4.2,4.3,4.4,4.5,4.6,4.7,4.8,5.0,5.1,5.2,5.3,5.4,5.5,5.6,5.7,5.8,5.9,6.0,6.1,6.2,6.3,6.4,6.5,6.6,7.0,7.1,7.2,7.3,7.4,7.5,8.0,8.1,8.2,8.3,9.0,9.1,9.2,9.3,9.4,9.5,9.6,9.7,10.0,10.1,10.2,10.3,10.4,11.0,11.1,11.2,11.3,11.4,11.5,11.6,12.0,12.1,12.2,12.3,12.4,12.5,13.0,13.1,13.2,13.3,13.4,13.5,14.0,14.1,14.2,14.3,14.4,14.5,14.6,15.0,15.1'. Please refer to https://aka.ms/DeploymentScriptsTroubleshoot for more deployment script information.\\\\\\\"\\\\r\\\\n                      }\\\\r\\\\n                    ]\\\\r\\\\n                  }\\\\r\\\\n                ]\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"BadRequest\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"InvalidTemplateDeployment\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The template deployment 'frontend-cert-uxl4bs2oechw4' is not valid according to the validation procedure. The tracking id is 'cfe19b22-d56a-4e05-ab80-54b3c9aba2a9'. See inner errors for details.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentPreflightValidationFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deploymentScripts/newCertwithRotationKV-frontend-cert-prow-j2736256\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"The script language version '15.3.0' is not supported. Supported versions: '2.7,2.8,3.0,3.1,3.2,3.3,3.4,3.5,3.6,3.7,3.8,4.1,4.2,4.3,4.4,4.5,4.6,4.7,4.8,5.0,5.1,5.2,5.3,5.4,5.5,5.6,5.7,5.8,5.9,6.0,6.1,6.2,6.3,6.4,6.5,6.6,7.0,7.1,7.2,7.3,7.4,7.5,8.0,8.1,8.2,8.3,9.0,9.1,9.2,9.3,9.4,9.5,9.6,9.7,10.0,10.1,10.2,10.3,10.4,11.0,11.1,11.2,11.3,11.4,11.5,11.6,12.0,12.1,12.2,12.3,12.4,12.5,13.0,13.1,13.2,13.3,13.4,13.5,14.0,14.1,14.2,14.3,14.4,14.5,14.6,15.0,15.1'. Please refer to https://aka.ms/DeploymentScriptsTroubleshoot for more deployment script information.\\\\\\\"\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"BadRequest\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"InvalidTemplateDeployment\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The template deployment 'admin-api-cert-uxl4bs2oechw4' is not valid according to the validation procedure. The tracking id is '3c53241c-9cfb-4e71-8f46-0fe5ea8a07f6'. See inner errors for details.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentPreflightValidationFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.Resources/deploymentScripts/newCertwithRotationKV-admin-api-cert-prow-j2736256\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"The script language version '15.3.0' is not supported. Supported versions: '2.7,2.8,3.0,3.1,3.2,3.3,3.4,3.5,3.6,3.7,3.8,4.1,4.2,4.3,4.4,4.5,4.6,4.7,4.8,5.0,5.1,5.2,5.3,5.4,5.5,5.6,5.7,5.8,5.9,6.0,6.1,6.2,6.3,6.4,6.5,6.6,7.0,7.1,7.2,7.3,7.4,7.5,8.0,8.1,8.2,8.3,9.0,9.1,9.2,9.3,9.4,9.5,9.6,9.7,10.0,10.1,10.2,10.3,10.4,11.0,11.1,11.2,11.3,11.4,11.5,11.6,12.0,12.1,12.2,12.3,12.4,12.5,13.0,13.1,13.2,13.3,13.4,13.5,14.0,14.1,14.2,14.3,14.4,14.5,14.6,15.0,15.1'. Please refer to https://aka.ms/DeploymentScriptsTroubleshoot for more deployment script information.\\\\\\\"\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a283537221b2b792b03ba5ce093eae9a42971d222da4b3f3d5cdf8cac98f7bde","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentPreflightValidationFailed; detail message The script language version '15.3.0' is not supported. Supported versions: '2.7,2.8,3.0,3.1,3.2,3.3,3.4,3.5,3.6,3.7,3.8,4.1,4.2,4.3,4.4,4.5,4.6,4.7,4.8,5.0,5.1,5.2,5.3,5.4,5.5,5.6,5.7,5.8,5.9,6.0,6.1,6.2,6.3,6.4,6.5,6.6,7.0,7.1,7.2,7.3,7.4,7.5,8.0,8.1,8.2,8.3,9.0,9.1,9.2,9.3,9.4,9.5,9.6,9.7,10.0,10.1,10.2,10.3,10.4,11.0,11.1,11.2,11.3,11.4,11.5,11.6,12.0,12.1,12.2,12.3,12.4,12.5,13.0,13.1,13.2,13.3,13.4,13.5,14.0,14.1,14.2,14.3,14.4,14.5,14.6,15.0,15.1'. Please","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a283537221b2b792b03ba5ce093eae9a42971d222da4b3f3d5cdf8cac98f7bde"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4212/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026747809713098752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4212,"pr_state":"closed","pr_sha":"395fd8316ddbcee82632b1ac8c16734e64825f81","final_merged_sha":"551e259a36be7a8b6e5199c23a5f2890117fae0f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T19:39:37Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"87c48f6323e0df8f4212f8e4d460b53d01d1cbba52586a68bcfbe3cbdd9b27e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4212/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026747809713098752","occurred_at":"2026-02-25T19:39:37Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026740502031765504","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4033,"pr_state":"closed","pr_sha":"79f03ffcb0ca76de3a7431540b5a6bd661a10cec","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T19:26:05Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"0b531f1a351aae27377b97dff64ec77e0c74dd1fa64c71344210439d82105859","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026740502031765504","occurred_at":"2026-02-25T19:26:05Z","failed_at":"provision","signature_id":"8502a51bc863354d73982a77213058401ba8e85e4e534c9c54c944ad29110fef","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-velero-image","test_suite":"step graph","failure_text":"time=2026-02-25T20:18:13.404Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image description=\"Step mirror-velero-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e velero.server.registry \u003cnil\u003e}:{\u003cnil\u003e velero.server.repository \u003cnil\u003e}@{\u003cnil\u003e velero.server.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T20:18:16.373Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.redhat.io/oadp/oadp-velero-rhel9@sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b to arohcpsvcdev.azurecr.io/oadp/oadp-velero-rhel9:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b.\\nThe image will still be available under it's original digest sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b in the target registry.\\nError response from registry: failed to resolve sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-rhel9/manifests/sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223","failure_pattern":"Error response from registry: failed to resolve sha256:\u003chex\u003e: GET \"\u003curl\u003e\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: \u003curl\u003e","search_query":"Error response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4206/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026738346327281664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4206,"pr_state":"closed","pr_sha":"b1cdfeb902184c5da685543ea4752d15f4fedbf3","final_merged_sha":"b1cdfeb902184c5da685543ea4752d15f4fedbf3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-25T19:17:31Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b6e99e585a50024a7ea15a712a69dd80f5eea91640a19bd853b17cf1472e8070","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4206/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026738346327281664","occurred_at":"2026-02-25T19:17:31Z","failed_at":"provision","signature_id":"dc988d340dad9de51c7b30f776b40dac5ac8a9e7fddd9a4a2a84de672db2304c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-hypershift-plugin-image","test_suite":"step graph","failure_text":"time=2026-02-25T20:14:00.044Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-hypershift-plugin-image description=\"Step mirror-hypershift-plugin-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e velero.hypershiftPlugin.registry \u003cnil\u003e}:{\u003cnil\u003e velero.hypershiftPlugin.repository \u003cnil\u003e}@{\u003cnil\u003e velero.hypershiftPlugin.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T20:14:04.485Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-hypershift-plugin-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.redhat.io/oadp/oadp-hypershift-velero-plugin-rhel9@sha256:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e to arohcpsvcdev.azurecr.io/oadp/oadp-hypershift-velero-plugin-rhel9:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e.\\nThe image will still be available under it's original digest sha256:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e in the target registry.\\nError response from registry: failed to resolve sha256:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-hypershift-velero-plugin-rhel9/manifests/sha256:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223","failure_pattern":"Error response from registry: failed to resolve sha256:\u003chex\u003e: GET \"\u003curl\u003e\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: \u003curl\u003e","search_query":"Error response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3094/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026738169436704768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3094,"pr_state":"closed","pr_sha":"4bea39ee41791d53fde9159d127b03ba09c007f9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T19:16:49Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"7acf2dc47342090522b4acf14fed1db01538d5e657d280d638ee58944cc41c26","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3094/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026738169436704768","occurred_at":"2026-02-25T19:16:49Z","failed_at":"provision","signature_id":"12e4c71a9a2f27c3933827b33d6567a5995c280323c72629a422bc5c75fcc5f8","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-azure-plugin-image","test_suite":"step graph","failure_text":"time=2026-02-25T20:19:25.705Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-azure-plugin-image description=\"Step mirror-azure-plugin-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e velero.azurePlugin.registry \u003cnil\u003e}:{\u003cnil\u003e velero.azurePlugin.repository \u003cnil\u003e}@{\u003cnil\u003e velero.azurePlugin.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T20:19:28.729Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-azure-plugin-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.redhat.io/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9@sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e to arohcpsvcdev.azurecr.io/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e.\\nThe image will still be available under it's original digest sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e in the target registry.\\nError response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223","failure_pattern":"Error response from registry: failed to resolve sha256:\u003chex\u003e: GET \"\u003curl\u003e\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: \u003curl\u003e","search_query":"Error response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4207/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026736595570266112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4207,"pr_state":"closed","pr_sha":"a02b279a61587d577a594a03ad7cf24d793edffe","final_merged_sha":"a02b279a61587d577a594a03ad7cf24d793edffe","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-25T19:10:33Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"adbb5c29eae1bc83988e14d9698eebf22936cfa1ca7b319f385f9b740fefe278","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4207/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026736595570266112","occurred_at":"2026-02-25T19:10:33Z","failed_at":"provision","signature_id":"8502a51bc863354d73982a77213058401ba8e85e4e534c9c54c944ad29110fef","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-velero-image","test_suite":"step graph","failure_text":"time=2026-02-25T20:04:24.765Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image description=\"Step mirror-velero-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e velero.server.registry \u003cnil\u003e}:{\u003cnil\u003e velero.server.repository \u003cnil\u003e}@{\u003cnil\u003e velero.server.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T20:04:27.592Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.redhat.io/oadp/oadp-velero-rhel9@sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b to arohcpsvcdev.azurecr.io/oadp/oadp-velero-rhel9:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b.\\nThe image will still be available under it's original digest sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b in the target registry.\\nError response from registry: failed to resolve sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-rhel9/manifests/sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223","failure_pattern":"Error response from registry: failed to resolve sha256:\u003chex\u003e: GET \"\u003curl\u003e\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: \u003curl\u003e","search_query":"Error response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4213/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026732791940517888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4213,"pr_state":"closed","pr_sha":"60d3a82dbbb6f19fa9811045ea4f586e748835c3","final_merged_sha":"096bc31775b32d485c1da66c7c477745a134e33c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T18:55:27Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"46df8ecdc3efdcbf4fd2caa8df8f0bd4964af2f377ef3634ebee62388955b191","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4213/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026732791940517888","occurred_at":"2026-02-25T18:55:27Z","failed_at":"provision","signature_id":"12e4c71a9a2f27c3933827b33d6567a5995c280323c72629a422bc5c75fcc5f8","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-azure-plugin-image","test_suite":"step graph","failure_text":"time=2026-02-25T19:40:13.244Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-azure-plugin-image description=\"Step mirror-azure-plugin-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e velero.azurePlugin.registry \u003cnil\u003e}:{\u003cnil\u003e velero.azurePlugin.repository \u003cnil\u003e}@{\u003cnil\u003e velero.azurePlugin.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T19:40:15.943Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-azure-plugin-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.redhat.io/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9@sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e to arohcpsvcdev.azurecr.io/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e.\\nThe image will still be available under it's original digest sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e in the target registry.\\nError response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223","failure_pattern":"Error response from registry: failed to resolve sha256:\u003chex\u003e: GET \"\u003curl\u003e\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: \u003curl\u003e","search_query":"Error response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4211/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026725922870136832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4211,"pr_state":"open","pr_sha":"b8334fea4a3ad4e205721e1236da749ba855a00b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T18:28:06Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"4ad7eae4aaf0fa717f16b6cef569b6eac0fc6d25c781c9f97b56a4c9641bac88","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4211/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026725922870136832","occurred_at":"2026-02-25T18:28:06Z","failed_at":"provision","signature_id":"8502a51bc863354d73982a77213058401ba8e85e4e534c9c54c944ad29110fef","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-velero-image","test_suite":"step graph","failure_text":"time=2026-02-25T19:16:56.156Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image description=\"Step mirror-velero-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e velero.server.registry \u003cnil\u003e}:{\u003cnil\u003e velero.server.repository \u003cnil\u003e}@{\u003cnil\u003e velero.server.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T19:16:58.639Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.redhat.io/oadp/oadp-velero-rhel9@sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b to arohcpsvcdev.azurecr.io/oadp/oadp-velero-rhel9:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b.\\nThe image will still be available under it's original digest sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b in the target registry.\\nError response from registry: failed to resolve sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-rhel9/manifests/sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223","failure_pattern":"Error response from registry: failed to resolve sha256:\u003chex\u003e: GET \"\u003curl\u003e\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: \u003curl\u003e","search_query":"Error response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026720520417841152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3679,"pr_state":"closed","pr_sha":"3ca804d0ed17f6519486da5d5d95366c60fc5e40","final_merged_sha":"3ca804d0ed17f6519486da5d5d95366c60fc5e40","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-25T18:06:41Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"61195ccce92a1b15346b184431e0564efbcf79d3847a8c44fae45a7974bedc75","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026720520417841152","occurred_at":"2026-02-25T18:06:41Z","failed_at":"provision","signature_id":"0cfdbc69e36cd9409140a1d94f3eca54c19e2976278c5f5c45b1651fb5c59fcf","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster","test_suite":"step graph","failure_text":"time=2026-02-25T18:47:54.409Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-02-25T18:47:55.212Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster\ntime=2026-02-25T18:47:56.787Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster deployment=f80e6f8b3407980613db5c04e9a30af4573312f0057b17fbc2b8f5eab928520f\ntime=2026-02-25T18:57:29.381Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7841152-mgmt-1/providers/Microsoft.Resources/deployments/f80e6f8b3407980613db5c04e9a30af4573312f0057b17fbc2b8f5eab928520f/operationStatuses/08584295616096671417\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j7841152-mgmt-1/providers/Microsoft.Resources/deployments/cluster-gu7qpftcdii5y\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ConcurrentFederatedIdentityCredentialsWritesForSingleManagedIdentity\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Too many Federated Identity Credentials are written concurrently for the managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j7841152-mgmt-1/providers/microsoft.managedidentity/userassignedidentities/image-puller'. Concurrent Federated Identity Credentials writes under the same managed identity are not supported.\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c28a1c9a31a25f8de598cf472ae1b520a4eed5de203b8d4a8f947873f64a07","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ConcurrentFederatedIdentityCredentialsWritesForSingleManagedIdentity; detail message Too many Federated Identity Credentials are written concurrently for the managed identity '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/microsoft.managedidentity/userassignedidentities/image-puller'. Concurrent Federated Identity Credentials writes under the same managed identity are not supported.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["07c28a1c9a31a25f8de598cf472ae1b520a4eed5de203b8d4a8f947873f64a07"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3094/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026719696165801984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3094,"pr_state":"closed","pr_sha":"4bea39ee41791d53fde9159d127b03ba09c007f9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T18:03:24Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ee2be5911b5c910e345008e6f98a36c2f97a00d623b5bb2922450469a06b8d40","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3094/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026719696165801984","occurred_at":"2026-02-25T18:03:24Z","failed_at":"provision","signature_id":"8502a51bc863354d73982a77213058401ba8e85e4e534c9c54c944ad29110fef","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-velero-image","test_suite":"step graph","failure_text":"time=2026-02-25T18:55:03.525Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image description=\"Step mirror-velero-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e velero.server.registry \u003cnil\u003e}:{\u003cnil\u003e velero.server.repository \u003cnil\u003e}@{\u003cnil\u003e velero.server.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T18:55:06.439Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.redhat.io/oadp/oadp-velero-rhel9@sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b to arohcpsvcdev.azurecr.io/oadp/oadp-velero-rhel9:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b.\\nThe image will still be available under it's original digest sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b in the target registry.\\nError response from registry: failed to resolve sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-rhel9/manifests/sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223","failure_pattern":"Error response from registry: failed to resolve sha256:\u003chex\u003e: GET \"\u003curl\u003e\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: \u003curl\u003e","search_query":"Error response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026719002784436224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3094,"pr_state":"closed","pr_sha":"4bea39ee41791d53fde9159d127b03ba09c007f9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T18:00:39Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"12ff9ed134356fa62a3551273130da9d3a1b171fc4979bcd6825267886be70a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026719002784436224","occurred_at":"2026-02-25T18:00:39Z","failed_at":"provision","signature_id":"12e4c71a9a2f27c3933827b33d6567a5995c280323c72629a422bc5c75fcc5f8","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-azure-plugin-image","test_suite":"step graph","failure_text":"time=2026-02-25T18:50:20.927Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-azure-plugin-image description=\"Step mirror-azure-plugin-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e velero.azurePlugin.registry \u003cnil\u003e}:{\u003cnil\u003e velero.azurePlugin.repository \u003cnil\u003e}@{\u003cnil\u003e velero.azurePlugin.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T18:50:23.621Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-azure-plugin-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.redhat.io/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9@sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e to arohcpsvcdev.azurecr.io/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e.\\nThe image will still be available under it's original digest sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e in the target registry.\\nError response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223","failure_pattern":"Error response from registry: failed to resolve sha256:\u003chex\u003e: GET \"\u003curl\u003e\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: \u003curl\u003e","search_query":"Error response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4212/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026718937386848256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4212,"pr_state":"closed","pr_sha":"395fd8316ddbcee82632b1ac8c16734e64825f81","final_merged_sha":"551e259a36be7a8b6e5199c23a5f2890117fae0f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T18:00:23Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"4fa8e47aaa242971a71f7d8fc492a700e7ad4cb9f74cf601195a1740a0dba904","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4212/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026718937386848256","occurred_at":"2026-02-25T18:00:23Z","failed_at":"provision","signature_id":"12e4c71a9a2f27c3933827b33d6567a5995c280323c72629a422bc5c75fcc5f8","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-azure-plugin-image","test_suite":"step graph","failure_text":"time=2026-02-25T18:50:19.854Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-azure-plugin-image description=\"Step mirror-azure-plugin-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e velero.azurePlugin.registry \u003cnil\u003e}:{\u003cnil\u003e velero.azurePlugin.repository \u003cnil\u003e}@{\u003cnil\u003e velero.azurePlugin.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T18:50:22.789Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-azure-plugin-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.redhat.io/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9@sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e to arohcpsvcdev.azurecr.io/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e.\\nThe image will still be available under it's original digest sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e in the target registry.\\nError response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223","failure_pattern":"Error response from registry: failed to resolve sha256:\u003chex\u003e: GET \"\u003curl\u003e\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: \u003curl\u003e","search_query":"Error response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026718293422772224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3094,"pr_state":"closed","pr_sha":"4bea39ee41791d53fde9159d127b03ba09c007f9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T17:57:50Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"735e897214f5d5ebf064e61f618b6fca55546ca17b34fd372ab4b0e5e3390cbf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026718293422772224","occurred_at":"2026-02-25T17:57:50Z","failed_at":"provision","signature_id":"dc988d340dad9de51c7b30f776b40dac5ac8a9e7fddd9a4a2a84de672db2304c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-hypershift-plugin-image","test_suite":"step graph","failure_text":"time=2026-02-25T18:55:38.447Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-hypershift-plugin-image description=\"Step mirror-hypershift-plugin-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e velero.hypershiftPlugin.registry \u003cnil\u003e}:{\u003cnil\u003e velero.hypershiftPlugin.repository \u003cnil\u003e}@{\u003cnil\u003e velero.hypershiftPlugin.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T18:55:41.126Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-hypershift-plugin-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.redhat.io/oadp/oadp-hypershift-velero-plugin-rhel9@sha256:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e to arohcpsvcdev.azurecr.io/oadp/oadp-hypershift-velero-plugin-rhel9:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e.\\nThe image will still be available under it's original digest sha256:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e in the target registry.\\nError response from registry: failed to resolve sha256:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-hypershift-velero-plugin-rhel9/manifests/sha256:f47f2f9099b6ad235ea1187556dc165aef725bcbe2b818ab62bb186ad485961e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223","failure_pattern":"Error response from registry: failed to resolve sha256:\u003chex\u003e: GET \"\u003curl\u003e\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: \u003curl\u003e","search_query":"Error response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4210/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026715525022748672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4210,"pr_state":"open","pr_sha":"0c3f47cd6898d3b4ee7c6525a5897c1eca6a7ad4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T17:46:50Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"f23e36ca9ddcb72f9721fb5008fbadd8a7961770870e011e97dcdb4a06081654","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4210/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026715525022748672","occurred_at":"2026-02-25T17:46:50Z","failed_at":"provision","signature_id":"8502a51bc863354d73982a77213058401ba8e85e4e534c9c54c944ad29110fef","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-velero-image","test_suite":"step graph","failure_text":"time=2026-02-25T18:43:32.187Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image description=\"Step mirror-velero-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e velero.server.registry \u003cnil\u003e}:{\u003cnil\u003e velero.server.repository \u003cnil\u003e}@{\u003cnil\u003e velero.server.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T18:43:35.528Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.redhat.io/oadp/oadp-velero-rhel9@sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b to arohcpsvcdev.azurecr.io/oadp/oadp-velero-rhel9:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b.\\nThe image will still be available under it's original digest sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b in the target registry.\\nError response from registry: failed to resolve sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-rhel9/manifests/sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223","failure_pattern":"Error response from registry: failed to resolve sha256:\u003chex\u003e: GET \"\u003curl\u003e\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: \u003curl\u003e","search_query":"Error response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026713351391809536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3925,"pr_state":"open","pr_sha":"4e06eacf5bb98f5ce156427cfc2c1a56f14220eb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T17:38:12Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"1915e79a1c36e73a019436d4afbddf652b5160332ace67cdb5b50e0be8ba5344","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3925/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026713351391809536","occurred_at":"2026-02-25T17:38:12Z","failed_at":"provision","signature_id":"8502a51bc863354d73982a77213058401ba8e85e4e534c9c54c944ad29110fef","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-velero-image","test_suite":"step graph","failure_text":"time=2026-02-25T18:31:33.569Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image description=\"Step mirror-velero-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e velero.server.registry \u003cnil\u003e}:{\u003cnil\u003e velero.server.repository \u003cnil\u003e}@{\u003cnil\u003e velero.server.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T18:31:36.356Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry registry.redhat.io from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.redhat.io/oadp/oadp-velero-rhel9@sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b to arohcpsvcdev.azurecr.io/oadp/oadp-velero-rhel9:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b.\\nThe image will still be available under it's original digest sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b in the target registry.\\nError response from registry: failed to resolve sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-rhel9/manifests/sha256:8ee2114096549e96bce5974584d14018c85e69bb8748588a89b960fdf502f64b\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223","failure_pattern":"Error response from registry: failed to resolve sha256:\u003chex\u003e: GET \"\u003curl\u003e\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: \u003curl\u003e","search_query":"Error response from registry: failed to resolve sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e: GET \\\"https://registry.redhat.io/v2/oadp/oadp-velero-plugin-for-microsoft-azure-rhel9/manifests/sha256:47a1a93a04208748c29073c17855aeca8f72ff2025d7f404b054abd375c8172e\\\": unauthorized: Please login to the Red Hat Registry using your Customer Portal credentials. Further instructions can be found here: https://access.redhat.com/RegistryAuthentication"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["326f73b54ddd360dd0d2ef88cb96af8416fee05758e17b4aaf63f865f51a3223"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4196/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026708589468454912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4196,"pr_state":"closed","pr_sha":"5a03840502dfd5db86eca68bfb8e212bff6f84ab","final_merged_sha":"b541ca7e953f8e1929cb1cc01938e54152b9c8e2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T17:19:16Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"bcc54dce640b8cb00409c4565007d702ebd836bc1971401fec143bbc14c4d83e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4196/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026708589468454912","occurred_at":"2026-02-25T17:19:16Z","failed_at":"provision","signature_id":"5259251d7d7352e4822ed036e9b5e3f561171ca010aa2c4b92b812bc233d1653","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","test_suite":"step graph","failure_text":"time=2026-02-25T17:55:59.092Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-02-25T17:56:01.248Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-02-25T17:56:02.425Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=41f16ef5151a15d838a02a56a0559f4405c126b36b8928aa7e037a1e9efa87dc\ntime=2026-02-25T17:57:33.227Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-prow-j8454912/providers/Microsoft.Resources/deployments/41f16ef5151a15d838a02a56a0559f4405c126b36b8928aa7e037a1e9efa87dc/operationStatuses/08584295647237856843\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8454912/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-prow-j8454912/providers/Microsoft.EventGrid/namespaces/arohcp-prow-maestro-j8454912\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report b86e066b-493f-4f0c-a6b1-b5f0bcb7d93f:2/25/2026 5:57:10 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4206/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026695339821502464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4206,"pr_state":"closed","pr_sha":"b1cdfeb902184c5da685543ea4752d15f4fedbf3","final_merged_sha":"b1cdfeb902184c5da685543ea4752d15f4fedbf3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-25T16:26:37Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"cb06bb37890e82575c3cc17a2ed2c7ea1f2b53fd44372153477bf5322dc776b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4206/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026695339821502464","occurred_at":"2026-02-25T16:26:37Z","failed_at":"e2e","signature_id":"ecbff54ceede268bb97c6ca20fed6f794c7aa4d36478afa100a80281fab62969","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"github.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:391\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0xc000f51680?, {0x7f2a65373660?, 0xc0009f4120?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:603 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0xc000d9ed80, {0x7f2a65373660, 0xc0009f4120}, {0x7698547, 0x15}, {0xc000014109, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:367 +0x2a5\ngithub.com/Azure/ARO-HCP/test/e2e.init.func17.2({0x7f2a65373660, 0xc0009f4120})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:67 +0xe9\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8076598?, 0xc0009f4120?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 79\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:391]: Test Panicked: failed to get subscription ID: AzureCLICredential: signal: killed\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:391]: Test Panicked\ngithub.com/Azure/ARO-HCP/test/util/framework.Must[...](...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:391\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).GetARMResourcesClientFactoryOrDie(0xc000f51680?, {0x7f2a65373660?, 0xc0009f4120?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:603 +0x37\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).NewResourceGroup(0xc000d9ed80, {0x7f2a65373660, 0xc0009f4120}, {0x7698547, 0x15}, {0xc000014109, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:367 +0x2a5\ngithub.com/Azure/ARO-HCP/test/e2e.init.func17.2({0x7f2a65373660, 0xc0009f4120})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:67 +0xe9\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8076598?, 0xc0009f4120?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 79\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:391]: Test Panicked: failed to get subscription ID: AzureCLICredential: signal: killed\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:391]: Test Panicked","failure_pattern_match":{"status":"clustered","failure_pattern_id":"39c6c5ed08e8e90ee04db2b87d32b29b6f38089eb34d7f09ee21959da678ff0e","failure_pattern":"Test Panicked","search_query":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_invocation_framework.go:391]: Test Panicked"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["39c6c5ed08e8e90ee04db2b87d32b29b6f38089eb34d7f09ee21959da678ff0e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4208/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026693341910929408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4208,"pr_state":"closed","pr_sha":"c8160d3983282ee9fdf3e2f47f59530a011c9e1c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-25T16:18:41Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026691483368689664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4033,"pr_state":"closed","pr_sha":"805cfcf4e811794a2d1d3b5ee72c56e84cab77e9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T16:11:18Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e4b904ea2696eb039a9b7dfde7278f5befd0244058f4e1d0dab2eeaa6a2c9bbd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026691483368689664","occurred_at":"2026-02-25T16:11:18Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026686729431289856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4119,"pr_state":"closed","pr_sha":"67242686301859b9fe72cdebee97795e7c971725","final_merged_sha":"8700ec5303d9646d72fd5761c2ccc9d31b35c5d0","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-25T15:52:24Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026683753530658816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3679,"pr_state":"closed","pr_sha":"3ca804d0ed17f6519486da5d5d95366c60fc5e40","final_merged_sha":"3ca804d0ed17f6519486da5d5d95366c60fc5e40","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-25T15:40:30Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"12e922d7ced4e9fce2b03b07a5acd09f7c1061350865d886ff410b4377de5bba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026683753530658816","occurred_at":"2026-02-25T15:40:30Z","failed_at":"provision","signature_id":"1e3c66f1b3aaa8e9531a0447e5fe758d4cfa54cfe1d96c0e23b03169e36088b8","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/istio-upgrade","test_suite":"step graph","failure_text":"time=2026-02-25T16:32:09.371Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade description=\"Step istio-upgrade\\n  Kind: Shell\\n  Command: ./istio.sh\\n\"\ntime=2026-02-25T16:32:18.368Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=istio-upgrade err=\"error running Shell Step, failed to execute shell command: + set -euo pipefail\\n+ ISTIO_NAMESPACE=aks-istio-system\\n+ echo '********** Check istio is up and running **************'\\n********** Check istio is up and running **************\\n++ kubectl get pods -n aks-istio-system -l istio.io/rev=asm-1-26 --field-selector=status.phase=Running --no-headers\\n++ wc -l\\n+ ISTIO_PODS_COUNT=2\\n+ [[ 2 -lt 2 ]]\\n+ echo '********** Download istioctl **************'\\n********** Download istioctl **************\\n++ uname\\n+ OS=Linux\\n+ '[' Linux = Darwin ']'\\n+ OSEXT=linux\\n++ uname -m\\n+ LOCAL_ARCH=x86_64\\n+ case \\\"${LOCAL_ARCH}\\\" in\\n+ ISTIO_ARCH=amd64\\n+ ISTIO_URL=https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz\\n+ SHA256_URL=https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz.sha256\\n+ curl -sL https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz -o istio-1.28.3-linux-amd64.tar.gz\\n+ curl -sL https://github.com/istio/istio/releases/download/1.28.3/istio-1.28.3-linux-amd64.tar.gz.sha256 -o istio-1.28.3-linux-amd64.tar.gz.sha256\\n+ sha256sum -c istio-1.28.3-linux-amd64.tar.gz.sha256\\nistio-1.28.3-linux-amd64.tar.gz: FAILED\\nsha256sum: WARNING: 1 computed checksum did NOT match\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"941f12df2ac8109e85b745c88ba5fd5783296c619a1efe2cf51485b6dc6bc270","failure_pattern":"istio-1.28.3-linux-amd64.tar.gz: FAILED","search_query":"istio-1.28.3-linux-amd64.tar.gz: FAILED"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["941f12df2ac8109e85b745c88ba5fd5783296c619a1efe2cf51485b6dc6bc270"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026681539160444928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4167,"pr_state":"closed","pr_sha":"5702af3e8e3af1de0bdf3bdffd21e229ad7a2381","final_merged_sha":"5702af3e8e3af1de0bdf3bdffd21e229ad7a2381","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-25T15:31:47Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026680796412121088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4167,"pr_state":"closed","pr_sha":"5702af3e8e3af1de0bdf3bdffd21e229ad7a2381","final_merged_sha":"5702af3e8e3af1de0bdf3bdffd21e229ad7a2381","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-25T15:28:50Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4207/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026671918379175936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4207,"pr_state":"closed","pr_sha":"a02b279a61587d577a594a03ad7cf24d793edffe","final_merged_sha":"a02b279a61587d577a594a03ad7cf24d793edffe","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-25T14:53:33Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4166/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026669840084766720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4166,"pr_state":"closed","pr_sha":"d0eeb47de9bc1e28370982dc1f2b5ba593f3322f","final_merged_sha":"e6b55da5d6cb3778ad30055d1acccf161634b854","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-25T14:45:18Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3764/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026667212424613888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3764,"pr_state":"closed","pr_sha":"a6ce4d3f31656feb7fa2b406b2fa44d4369a5d7d","final_merged_sha":"e5b4a5c5d47e76f3a52bc9ed42615eac10be20ff","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-25T14:34:51Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4004/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026656708431974400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4004,"pr_state":"open","pr_sha":"532d1afe6a01dde6532be157ce86561e504f9413","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-25T13:53:07Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4189/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026656590270042112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4189,"pr_state":"open","pr_sha":"cbb5a982644bae16afb394f00697943175bf9496","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-25T13:52:39Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3913/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026653947602669568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3913,"pr_state":"open","pr_sha":"2b1240006f2cc75b5e9c5365f1d11dd559ba55d8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-25T13:42:08Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4166/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026653134079660032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4166,"pr_state":"closed","pr_sha":"8077c8ce8f6dde43e991590e5a2baa86fb3d6ae8","final_merged_sha":"e6b55da5d6cb3778ad30055d1acccf161634b854","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T13:38:55Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f27b8f13e374b4852b6f52b760ab64e06f63b6a88a6453ae6a7d1f5615df3c3a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4166/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026653134079660032","occurred_at":"2026-02-25T13:38:55Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026649569168723968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4032,"pr_state":"closed","pr_sha":"4ddffef6731636e9d4a6f7f53b1faeb109bcab53","final_merged_sha":"4ddffef6731636e9d4a6f7f53b1faeb109bcab53","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-25T13:24:45Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4166/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026649190469210112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4166,"pr_state":"closed","pr_sha":"197dda7417273193cbd8afa63ff78707a91e8032","final_merged_sha":"e6b55da5d6cb3778ad30055d1acccf161634b854","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T13:23:14Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"fcf8e0c950ee992d0de009349c8cd21da2d75b9abaa24793d2d56b58aae73ce7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4166/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026649190469210112","occurred_at":"2026-02-25T13:23:14Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3094/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026644841177288704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3094,"pr_state":"closed","pr_sha":"4bea39ee41791d53fde9159d127b03ba09c007f9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-25T13:05:52Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4205/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026641876928434176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4205,"pr_state":"closed","pr_sha":"592efb2f1f9584761c4d09ddb82a6d709c973b36","final_merged_sha":"a36f37aae6ead63c15a10186a904f97e4e0fc50d","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-25T12:54:11Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4201/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026640066373226496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4201,"pr_state":"closed","pr_sha":"6a05d48878649bf6564f1e58c30a26055cd5cc0e","final_merged_sha":"6a05d48878649bf6564f1e58c30a26055cd5cc0e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-25T12:46:59Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4167/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026635532779393024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4167,"pr_state":"closed","pr_sha":"5702af3e8e3af1de0bdf3bdffd21e229ad7a2381","final_merged_sha":"5702af3e8e3af1de0bdf3bdffd21e229ad7a2381","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-25T12:28:58Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4204/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026634306754973696","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4204,"pr_state":"closed","pr_sha":"57040414c383f38c6df779151de1ebe8d6d2ac9b","final_merged_sha":"57040414c383f38c6df779151de1ebe8d6d2ac9b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-25T12:24:06Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4200/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026623572306300928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4200,"pr_state":"closed","pr_sha":"df979fb0b38ebd41e92cfe0036e254d507104d1d","final_merged_sha":"df979fb0b38ebd41e92cfe0036e254d507104d1d","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-25T11:41:27Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4056/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026606856713015296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4056,"pr_state":"closed","pr_sha":"2714126704381c47bf54267faeabfff8fcfe8378","final_merged_sha":"2714126704381c47bf54267faeabfff8fcfe8378","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-25T10:35:01Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026606123770974208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4056,"pr_state":"closed","pr_sha":"2714126704381c47bf54267faeabfff8fcfe8378","final_merged_sha":"2714126704381c47bf54267faeabfff8fcfe8378","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-25T10:32:07Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4075/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026588579458715648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4075,"pr_state":"closed","pr_sha":"aa0d5ab40e4cf36e213e9bb18cac9eba9ae6aba2","final_merged_sha":"4c157031af423b1f7a6117f86f65c7962bd89ff5","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T09:22:24Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"d2c4b15e261e13cd44854ef86e12164c177ba340c9030c4b067a3527ae131600","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4075/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026588579458715648","occurred_at":"2026-02-25T09:22:24Z","failed_at":"provision","signature_id":"d4ebee5d0600e942e753aaccfb2083f00edf485ece75d873bf54245138f6bf81","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.Backend/service/deploy","test_suite":"step graph","failure_text":"time=2026-02-25T10:13:11.423Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-02-25T10:13:12.107Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud maestro:map[sourceEnvironmentIdentifier:arohcpprow] msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj8715648.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj8715648.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:d79c2524-9c8f-4718-b6cc-c7bbb6e7de0a databaseName:arohcpprow-rp-j8715648 databaseUrl:https://arohcpprow-rp-j8715648.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-0j40hnqt/pipeline@sha256:ce193edf2450d70f9b032f2015c940f702b34f49906b7373048f1451f8ac4daa replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-0j40hnqt/pipeline:pull workloadIdentityClientId:3d10ae00-294d-4cdb-b665-2f56cfc8dddb workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:d79c2524-9c8f-4718-b6cc-c7bbb6e7de0a workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-02-25T10:13:12.107Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-25T10:13:12.354Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-02-25T10:13:12.354Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-02-25T10:13:12.354Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-25T10:13:12.711Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-25T10:18:19.347Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-02-25T10:18:19.347Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-25T10:18:19.347Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-25T10:18:22.143Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Release \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\" values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud maestro:map[sourceEnvironmentIdentifier:arohcpprow] msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj8715648.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj8715648.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:d79c2524-9c8f-4718-b6cc-c7bbb6e7de0a databaseName:arohcpprow-rp-j8715648 databaseUrl:https://arohcpprow-rp-j8715648.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-0j40hnqt/pipeline@sha256:ce193edf2450d70f9b032f2015c940f702b34f49906b7373048f1451f8ac4daa replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-0j40hnqt/pipeline:pull workloadIdentityClientId:3d10ae00-294d-4cdb-b665-2f56cfc8dddb workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:d79c2524-9c8f-4718-b6cc-c7bbb6e7de0a workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-02-25T10:18:22.145Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/4yTQY/aPhDF7/kUo5xAileQv/5VRbUHuu2p2gq1PbXqwbFfwYvjicYOiBUfvkpYSrQKLdfH7z3exDOZRyJB5FYMIt2T1UknXXlMuHqCSZ9csIuYxIV1QSfps67xR8IOoVdio81Znv7IiIjypZFV6/17F6wL67ygvNJmi2BV03qvqouuhdXGNHlxMq4AWbZpg5Cc0clxGEDqHFIjiTNxxP+FPTrZQ1uIgod5FXImP6DxfKgR0sg/jPBdKxb33JdasXfm0FPe8/5c6C/2rzCCtBLeOQt58DrG4WeJkJ0zUFsc1E63Po1G9MwjB5dYbmv94lkaw+1p0uvsA4dfbv2om2Ex/dwKlLQhuRrK9Mit3qt090oju/HvR3sZ58adyH6+y7ZtBQlIiB+7hY3ZkfYbCKibJyZdN1Qh7YFAE6sTOnmSl7PyjZqVqvz/23y2mP+3mJff8ynd3dFV5u2i7JhpdqQndoG2Lth7FwJkcGYc6HJdw7N6fU/ZkRrh7tdL0YIatif4tpCCPK8LqhGjXneRLBZC1WEwvEU0vwMAAP//MRvEEQwEAAA=\"\ntime=2026-02-25T10:18:22.147Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-25T10:18:22.147Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: InProgress\\ncontext deadline exceeded\"\ntime=2026-02-25T10:19:22.148Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-02-25T10:19:22.843Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud maestro:map[sourceEnvironmentIdentifier:arohcpprow] msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj8715648.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj8715648.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:d79c2524-9c8f-4718-b6cc-c7bbb6e7de0a databaseName:arohcpprow-rp-j8715648 databaseUrl:https://arohcpprow-rp-j8715648.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-0j40hnqt/pipeline@sha256:ce193edf2450d70f9b032f2015c940f702b34f49906b7373048f1451f8ac4daa replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-0j40hnqt/pipeline:pull workloadIdentityClientId:3d10ae00-294d-4cdb-b665-2f56cfc8dddb workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:d79c2524-9c8f-4718-b6cc-c7bbb6e7de0a workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-02-25T10:19:22.843Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-25T10:19:23.133Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-02-25T10:19:23.133Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-02-25T10:19:23.133Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-25T10:19:23.308Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=1\ntime=2026-02-25T10:24:29.268Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-02-25T10:24:29.268Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-25T10:24:29.268Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-25T10:24:33.348Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Upgrade \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\" values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud maestro:map[sourceEnvironmentIdentifier:arohcpprow] msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj8715648.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj8715648.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:d79c2524-9c8f-4718-b6cc-c7bbb6e7de0a databaseName:arohcpprow-rp-j8715648 databaseUrl:https://arohcpprow-rp-j8715648.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-0j40hnqt/pipeline@sha256:ce193edf2450d70f9b032f2015c940f702b34f49906b7373048f1451f8ac4daa replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-0j40hnqt/pipeline:pull workloadIdentityClientId:3d10ae00-294d-4cdb-b665-2f56cfc8dddb workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:d79c2524-9c8f-4718-b6cc-c7bbb6e7de0a workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-02-25T10:24:33.349Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/4yTQW8aMRCF7/srRnsK0joim7ZSqXJI056qVKjpqVUPXvsVHLye1dgLIuLHV4ZQVggUbqvnzzNv3noKj0SCyL0YRLojq5NOuvG44uYZJn1zwU5iEhdmFe2k77rFfwlLhK0SO2328uh3QURUPkGWzuDeGO5DKisqG20WCDZ/amE1N11Z7dgHDn/d7FF3A0zpl16gpA/JtVBmi1x69yz9gz2y7KEtRMHDJMfhBPnqf3Ci9tVbJHEmnrh0b2Tae//ZBevCbOio671XzUE/vtinOYt70dnNlL0z6y3lPa/2/dT5AJ9gBGkqvHQW8uB1jMPmcTeLWmCtlrr36UwyJ3y/HdQXdJ7XLUI6kdUJfgpIHhchOaOPil6Q8euPeeTgEssbPYs/n4pF30ACEuLX/GBjsaHVHALKDysm3XbUIK2AQFdWJ2T5qqzH9Qc1rlX9/ufNeHLzcVLf/ipHdH1N55j63eQ2M6NiQ8/sAi1csHcuBMhgzTjQYbuGa3W8T8WGOuF8ejBaUcd2B19WpCLPs4paxKhnuSSLhVCzHgxvEc2/AAAA//9WZwixDAQAAA==\"\ntime=2026-02-25T10:24:33.351Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-25T10:24:33.351Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-02-25T10:25:33.354Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-02-25T10:25:34.105Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud maestro:map[sourceEnvironmentIdentifier:arohcpprow] msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj8715648.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj8715648.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:d79c2524-9c8f-4718-b6cc-c7bbb6e7de0a databaseName:arohcpprow-rp-j8715648 databaseUrl:https://arohcpprow-rp-j8715648.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-0j40hnqt/pipeline@sha256:ce193edf2450d70f9b032f2015c940f702b34f49906b7373048f1451f8ac4daa replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-0j40hnqt/pipeline:pull workloadIdentityClientId:3d10ae00-294d-4cdb-b665-2f56cfc8dddb workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:d79c2524-9c8f-4718-b6cc-c7bbb6e7de0a workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-02-25T10:25:34.105Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-25T10:25:34.467Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy namespace=aro-hcp\ntime=2026-02-25T10:25:34.467Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy dryRun=false\ntime=2026-02-25T10:25:34.467Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-25T10:25:34.691Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy numVersions=2\ntime=2026-02-25T10:30:40.986Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-02-25T10:30:40.986Z level=INFO msg=\"Finished deploying Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-25T10:30:40.986Z level=INFO msg=\"Running inline diagnostics.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-25T10:30:45.535Z level=INFO msg=\"Determined release status.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy release=aro-hcp-backend-dev namespace=aro-hcp status=failed description=\"Upgrade \\\"aro-hcp-backend-dev\\\" failed: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\" values=\"map[azure:map[cloudEnvironmentName:AzurePublicCloud maestro:map[sourceEnvironmentIdentifier:arohcpprow] msiRpDataPlaneAudienceResource:https://dummy.org ocpAcrHostname:arohcpocpdev.azurecr.io ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev oidcIssuerBaseUrl:https://arohcpprowoidcj8715648.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpprowoidcj8715648.blob.core.windows.net/ tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesConfig:map[certificatesGenerationSource:AzureKeyVault issuer:Self]] clustersService:map[namespace:clusters-service serviceAccount:clusters-service] configMap:map[backendMiClientId:d79c2524-9c8f-4718-b6cc-c7bbb6e7de0a databaseName:arohcpprow-rp-j8715648 databaseUrl:https://arohcpprow-rp-j8715648.documents.azure.com:443/ location:westus3] deployment:map[imageName:arohcpsvcdev.azurecr.io/ci-op-0j40hnqt/pipeline@sha256:ce193edf2450d70f9b032f2015c940f702b34f49906b7373048f1451f8ac4daa replicas:2 zoneCount:4] fpa:map[certName:firstPartyCert2 clientId:b3cb2fab-15cb-4583-ad06-f91da9bfe2d1] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-0j40hnqt/pipeline:pull workloadIdentityClientId:3d10ae00-294d-4cdb-b665-2f56cfc8dddb workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceAccount:map[name:backend workloadIdentityClientId:d79c2524-9c8f-4718-b6cc-c7bbb6e7de0a workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] serviceKeyVault:map[name:aro-hcp-dev-svc-kv] tracing:map[address:http://ingest.observability:4318 exporter:otlp]]\"\ntime=2026-02-25T10:30:45.536Z level=INFO msg=\"Kube-events kusto link for troubleshooting:\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy url=\"https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs?query=H4sIAAAAAAAA/4yTMW/bMBCFd/2Kg6YYEAPXiTu4yOCmnYoURtMpRQeKfJUZU6RwPNlw4B9fyI5rwVAab8Lj946nd7zMQ4iRYssGie7IatGiS4+rWD7DyDcX7CwJu1AVdJC+6xr/JKwR9kpqtDnKo18ZEVH+CF47g7kxsQ2SF5SX2qwQbPepOaqlafLiwP6IHp9dsC5UPVB5aAtW8DDiYhgwvl7SO1FHcw1hZ9KAaQHwvJUlgjijzypf4H+EYciC49pZ8L3XKfW7Toee1Apbtdatl4ES9zH8cdWDbvpG/dIyFLdBXA1l9sil3jfpLtpOfj/LL2h83NYIMhDHAD83vGi9H5hb03qvypN+bmxlGdm97KNfRO/Mdk95HzfH2P9z7+vMH2JwEvmdXrPfn7JVW4IDBOlr92BTtqPNEgzqck6i64ZKyAYIdGW1oJOv8sl48lGNJ2oy/flhPJtMZze3T/mIrq/pLeZmPLudPuWjUbaj5+gCrVywdy4EcG/NYqDTdvXX6nyfsh01HLvTU6MFNdEe4MuKFORjVVCNlHTVlYxswVRuez9vkczfAAAA//8AiQ0nDAQAAA==\"\ntime=2026-02-25T10:30:45.538Z level=INFO msg=\"Deployment complete.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy\ntime=2026-02-25T10:30:45.538Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy step=deploy error=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"\ntime=2026-02-25T10:31:45.544Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.Backend resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9a3364ebecf8b32a7511e0c1a2f7a3bf0739c1b446124dca86eaa8a33f7d1fa","failure_pattern":"resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: resource not ready, name: aro-hcp-backend, kind: Deployment, status: Failed\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d9a3364ebecf8b32a7511e0c1a2f7a3bf0739c1b446124dca86eaa8a33f7d1fa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3764/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026583498982166528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3764,"pr_state":"closed","pr_sha":"9ed1eef6b784c4e41db885becacaee6b651728fb","final_merged_sha":"e5b4a5c5d47e76f3a52bc9ed42615eac10be20ff","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-25T09:02:12Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026583326675963904","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3679,"pr_state":"closed","pr_sha":"84c6ad38191d00677c97efbb0a2cf3f865bdd256","final_merged_sha":"3ca804d0ed17f6519486da5d5d95366c60fc5e40","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-25T09:01:30Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4075/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026579966774218753","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4075,"pr_state":"closed","pr_sha":"a13199e4dfd64c7b80451500108b8d02a11e3759","final_merged_sha":"4c157031af423b1f7a6117f86f65c7962bd89ff5","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T08:48:10Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"70088162cc9b106508607d752a310d5ad1f3974da9a94ac6edf81a3e5737241d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4075/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026579966774218753","occurred_at":"2026-02-25T08:48:10Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4203/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026576621175050240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4203,"pr_state":"closed","pr_sha":"6e62bfe70eb560912257d601ae530b6ea19c94cc","final_merged_sha":"6e62bfe70eb560912257d601ae530b6ea19c94cc","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-25T08:34:47Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4202/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026574659494875136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4202,"pr_state":"closed","pr_sha":"97f1689575c4f946bad21cae9b4b6652fe0a1ced","final_merged_sha":"97f1689575c4f946bad21cae9b4b6652fe0a1ced","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-25T08:26:55Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4196/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026570625941770240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4196,"pr_state":"closed","pr_sha":"dc5fc75443ff251e7405b679fe2ef4b9b48337fe","final_merged_sha":"b541ca7e953f8e1929cb1cc01938e54152b9c8e2","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-02-25T08:11:03Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4197/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026566180373794816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4197,"pr_state":"closed","pr_sha":"1b7a82bd41571111c5cc5d4741df5e2a6a30b12b","final_merged_sha":"1b7a82bd41571111c5cc5d4741df5e2a6a30b12b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-25T07:53:23Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4056/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026563542999633920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4056,"pr_state":"closed","pr_sha":"2714126704381c47bf54267faeabfff8fcfe8378","final_merged_sha":"2714126704381c47bf54267faeabfff8fcfe8378","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-25T07:42:54Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4201/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026563379656658944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4201,"pr_state":"closed","pr_sha":"6a05d48878649bf6564f1e58c30a26055cd5cc0e","final_merged_sha":"6a05d48878649bf6564f1e58c30a26055cd5cc0e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-02-25T07:42:16Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026562011004932096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3679,"pr_state":"closed","pr_sha":"e5ce62af9c3c278b3084bb9ef2dbeb2ded76a76d","final_merged_sha":"3ca804d0ed17f6519486da5d5d95366c60fc5e40","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T07:36:49Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"aaf7bb42864e00077b995616c6bf80eb4a2b22b1bfb3ae090296cc8044340302","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026562011004932096","occurred_at":"2026-02-25T07:36:49Z","failed_at":"provision","signature_id":"18810fff95baf7e43ee1c1b204f433a8bb133b824f960ef12beadc31740ca9b5","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-02-25T08:27:10.553Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e sessiongate.image.registry \u003cnil\u003e}:{\u003cnil\u003e sessiongate.image.repository \u003cnil\u003e}@{\u003cnil\u003e sessiongate.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T08:27:38.234Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nSetting up registry authentication for CI source registry.\\ninfo: Using registry public hostname registry.build06.ci.openshift.org\\nSaved credentials for registry.build06.ci.openshift.org into /tmp/tmp.ZkrpcQnKXQ/containers/auth.json\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.build06.ci.openshift.org/ci-op-6z71dbgb/pipeline@sha256:571a3a553a4b3fdb3bf4625d9a5f67b111107b53b5ca9c402ec663d2ba40a5ce to arohcpsvcdev.azurecr.io/ci-op-6z71dbgb/pipeline:571a3a553a4b3fdb3bf4625d9a5f67b111107b53b5ca9c402ec663d2ba40a5ce.\\nThe image will still be available under it's original digest sha256:571a3a553a4b3fdb3bf4625d9a5f67b111107b53b5ca9c402ec663d2ba40a5ce in the target registry.\\nError response from registry: generic storage error: unknown_storage_failure\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"51b4e43d29790aeda46b32c9bd3546efa6f1fc018eb5f35003f5ba847596c023","failure_pattern":"Error response from registry: generic storage error: unknown_storage_failure","search_query":"Error response from registry: generic storage error: unknown_storage_failure"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["51b4e43d29790aeda46b32c9bd3546efa6f1fc018eb5f35003f5ba847596c023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4200/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026560518935810048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4200,"pr_state":"closed","pr_sha":"df979fb0b38ebd41e92cfe0036e254d507104d1d","final_merged_sha":"df979fb0b38ebd41e92cfe0036e254d507104d1d","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-25T07:30:53Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"396694bfee7479bb6e9157fe1df373543dde92406b234c88c6fdd6e88abeb36c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4200/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026560518935810048","occurred_at":"2026-02-25T07:30:53Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4199/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026535181657051136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4199,"pr_state":"closed","pr_sha":"f357ca7a481d5b764a2dc781a9431af6f79c283e","final_merged_sha":"f357ca7a481d5b764a2dc781a9431af6f79c283e","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-25T05:50:13Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3509e2f9e1ce19be0ed0efa66a356696b379b518a1bb9c590cebc79e101cf755","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4199/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026535181657051136","occurred_at":"2026-02-25T05:50:13Z","failed_at":"provision","signature_id":"ef6be1eacd52d9c92f082d09ca6be08faf64a187059c037d1c14b6815f3b17a4","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.RP.HypershiftOperator/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-02-25T06:36:55.477Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.HypershiftOperator resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e hypershift.image.registry \u003cnil\u003e}:{\u003cnil\u003e hypershift.image.repository \u003cnil\u003e}@{\u003cnil\u003e hypershift.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T06:36:57.027Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.RP.HypershiftOperator resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nERROR: (Forbidden) Operation get is not allowed on a disabled secret.\\nCode: Forbidden\\nMessage: Operation get is not allowed on a disabled secret.\\nInner error: {\\n    \\\"code\\\": \\\"SecretDisabled\\\"\\n}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"00cc1690067acae3a7fcd4d3bbf945ca9b0a7ef086dbcf8c898dcc7314adc3cd","failure_pattern":"Inner error: {","search_query":"Inner error: {"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["00cc1690067acae3a7fcd4d3bbf945ca9b0a7ef086dbcf8c898dcc7314adc3cd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026523413610237952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4188,"pr_state":"closed","pr_sha":"9ff64c743d742d674acb9ecd209da84ed44f0ca5","final_merged_sha":"ad7003e5a941946bb5a1fed27c539b6ee3d95872","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T05:02:46Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"8422d4d80c68908a5251e43ef21c1f33e573573ea29cf32f3328c16c8f079f1b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026523413610237952","occurred_at":"2026-02-25T05:02:46Z","failed_at":"provision","signature_id":"5cfcf90a4656729f9ae14342a43b76f0b5dbd75490fb1ba03e8b013dd295ecac","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-azure-plugin-image","test_suite":"step graph","failure_text":"time=2026-02-25T05:21:51.649Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-azure-plugin-image description=\"Step mirror-azure-plugin-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e velero.azurePlugin.registry \u003cnil\u003e}:{\u003cnil\u003e velero.azurePlugin.repository \u003cnil\u003e}@{\u003cnil\u003e velero.azurePlugin.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T05:21:53.022Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-azure-plugin-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nERROR: (Forbidden) Operation get is not allowed on a disabled secret.\\nCode: Forbidden\\nMessage: Operation get is not allowed on a disabled secret.\\nInner error: {\\n    \\\"code\\\": \\\"SecretDisabled\\\"\\n}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"00cc1690067acae3a7fcd4d3bbf945ca9b0a7ef086dbcf8c898dcc7314adc3cd","failure_pattern":"Inner error: {","search_query":"Inner error: {"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["00cc1690067acae3a7fcd4d3bbf945ca9b0a7ef086dbcf8c898dcc7314adc3cd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026493038741688320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4188,"pr_state":"closed","pr_sha":"9ff64c743d742d674acb9ecd209da84ed44f0ca5","final_merged_sha":"ad7003e5a941946bb5a1fed27c539b6ee3d95872","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T03:02:45Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"ea014e933a1f342d442566045250460edf6e01e8bcba23fc0ac46f2f7a342b9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4188/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026493038741688320","occurred_at":"2026-02-25T03:02:45Z","failed_at":"provision","signature_id":"c0e73729f65c1bddce0d205afd5219d39a2909192e457ed4b8bdc151c1b7e673","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-02-25T03:54:32.710Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e secretSyncController.image.registry \u003cnil\u003e}:{\u003cnil\u003e secretSyncController.image.repository \u003cnil\u003e}@{\u003cnil\u003e secretSyncController.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T03:54:34.346Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry registry.k8s.io from arohcpdev-global KV.\\nERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8c9fc918c1d28a5fd2dc0ad245350a62ec83b0c2f810eac421c4c02c68447eb5","failure_pattern":"ERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4","search_query":"ERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8c9fc918c1d28a5fd2dc0ad245350a62ec83b0c2f810eac421c4c02c68447eb5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4134/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026479980782817280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4134,"pr_state":"closed","pr_sha":"9fdaef42a0663c4aa5d5cd32e40a46627c1d79fb","final_merged_sha":"9fdaef42a0663c4aa5d5cd32e40a46627c1d79fb","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-25T02:10:44Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c0fe498b6ad3f9c9b7e777c23492afcc997e7e02c7b5d8621b46823cbeda787e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4134/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026479980782817280","occurred_at":"2026-02-25T02:10:44Z","failed_at":"provision","signature_id":"0d06b6d1a561268f2553e0f12be67c3b1bd9affadf20bf8c5c24221c26608dc4","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-velero-image","test_suite":"step graph","failure_text":"time=2026-02-25T03:03:01.221Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image description=\"Step mirror-velero-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e velero.server.registry \u003cnil\u003e}:{\u003cnil\u003e velero.server.repository \u003cnil\u003e}@{\u003cnil\u003e velero.server.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T03:03:02.633Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8c9fc918c1d28a5fd2dc0ad245350a62ec83b0c2f810eac421c4c02c68447eb5","failure_pattern":"ERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4","search_query":"ERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8c9fc918c1d28a5fd2dc0ad245350a62ec83b0c2f810eac421c4c02c68447eb5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4167/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026468979786125312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4167,"pr_state":"closed","pr_sha":"5702af3e8e3af1de0bdf3bdffd21e229ad7a2381","final_merged_sha":"5702af3e8e3af1de0bdf3bdffd21e229ad7a2381","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-25T01:27:09Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"92d005c8582567c29799c3c75c10622f677f0a023b512f84fdde3700d8079ddd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4167/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026468979786125312","occurred_at":"2026-02-25T01:27:09Z","failed_at":"provision","signature_id":"ea46b0ad67c8379b266eee93fa1c2d9dc9b312971a02c5f1ba235eb69f35c7b4","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Server/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-02-25T02:17:47.824Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e maestro.image.registry \u003cnil\u003e}:{\u003cnil\u003e maestro.image.repository \u003cnil\u003e}@{\u003cnil\u003e maestro.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T02:17:49.114Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8c9fc918c1d28a5fd2dc0ad245350a62ec83b0c2f810eac421c4c02c68447eb5","failure_pattern":"ERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4","search_query":"ERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8c9fc918c1d28a5fd2dc0ad245350a62ec83b0c2f810eac421c4c02c68447eb5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026468315441926144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4167,"pr_state":"closed","pr_sha":"5702af3e8e3af1de0bdf3bdffd21e229ad7a2381","final_merged_sha":"5702af3e8e3af1de0bdf3bdffd21e229ad7a2381","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-25T01:24:30Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"aea5325acd4af88ec4acf99fa1ec986c2c4a2afa49ec5f3ba75a6fc597e383cd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026468315441926144","occurred_at":"2026-02-25T01:24:30Z","failed_at":"provision","signature_id":"cdf8d8e2d103a4b3ab1c2a6232912f6a711187ae189bab007a2056b3d88cbc25","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Agent/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-02-25T02:13:06.734Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e maestro.image.registry \u003cnil\u003e}:{\u003cnil\u003e maestro.image.repository \u003cnil\u003e}@{\u003cnil\u003e maestro.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T02:13:08.260Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8c9fc918c1d28a5fd2dc0ad245350a62ec83b0c2f810eac421c4c02c68447eb5","failure_pattern":"ERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4","search_query":"ERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8c9fc918c1d28a5fd2dc0ad245350a62ec83b0c2f810eac421c4c02c68447eb5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026467599168049152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3679,"pr_state":"closed","pr_sha":"600af2000421a6cfa8f9e2c0f0233f45d82b0dab","final_merged_sha":"3ca804d0ed17f6519486da5d5d95366c60fc5e40","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T01:21:40Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"724a1b776296c502c7bbfca19ab00af10698200386ab49a71b0be696e629a548","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/3679/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026467599168049152","occurred_at":"2026-02-25T01:21:40Z","failed_at":"provision","signature_id":"ea46b0ad67c8379b266eee93fa1c2d9dc9b312971a02c5f1ba235eb69f35c7b4","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Server/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-02-25T02:12:16.259Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e maestro.image.registry \u003cnil\u003e}:{\u003cnil\u003e maestro.image.repository \u003cnil\u003e}@{\u003cnil\u003e maestro.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T02:12:17.962Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Server resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8c9fc918c1d28a5fd2dc0ad245350a62ec83b0c2f810eac421c4c02c68447eb5","failure_pattern":"ERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4","search_query":"ERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8c9fc918c1d28a5fd2dc0ad245350a62ec83b0c2f810eac421c4c02c68447eb5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026465553534685184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":3679,"pr_state":"closed","pr_sha":"600af2000421a6cfa8f9e2c0f0233f45d82b0dab","final_merged_sha":"3ca804d0ed17f6519486da5d5d95366c60fc5e40","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T01:13:32Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"75a9440486bc8bb286dedbdde06c829683c3dce0dcf8826a24a7397d86b3443f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026465553534685184","occurred_at":"2026-02-25T01:13:32Z","failed_at":"provision","signature_id":"cdf8d8e2d103a4b3ab1c2a6232912f6a711187ae189bab007a2056b3d88cbc25","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Agent/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-02-25T01:56:44.376Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e maestro.image.registry \u003cnil\u003e}:{\u003cnil\u003e maestro.image.repository \u003cnil\u003e}@{\u003cnil\u003e maestro.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T01:56:46.584Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8c9fc918c1d28a5fd2dc0ad245350a62ec83b0c2f810eac421c4c02c68447eb5","failure_pattern":"ERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4","search_query":"ERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8c9fc918c1d28a5fd2dc0ad245350a62ec83b0c2f810eac421c4c02c68447eb5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026455224637460480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4169,"pr_state":"closed","pr_sha":"0c69fe57c5ead617ee2e4d611cf3cd4d35485901","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-02-25T00:32:29Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"f8760b22171e4fb43d6219abc17b51f799cfe0b9661e936084628ec5c8538a16","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026455224637460480","occurred_at":"2026-02-25T00:32:29Z","failed_at":"provision","signature_id":"0d06b6d1a561268f2553e0f12be67c3b1bd9affadf20bf8c5c24221c26608dc4","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Velero/global/mirror-velero-image","test_suite":"step graph","failure_text":"time=2026-02-25T01:19:21.853Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image description=\"Step mirror-velero-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e velero.server.registry \u003cnil\u003e}:{\u003cnil\u003e velero.server.repository \u003cnil\u003e}@{\u003cnil\u003e velero.server.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T01:19:23.238Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Velero resourceGroup=global step=mirror-velero-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8c9fc918c1d28a5fd2dc0ad245350a62ec83b0c2f810eac421c4c02c68447eb5","failure_pattern":"ERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4","search_query":"ERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8c9fc918c1d28a5fd2dc0ad245350a62ec83b0c2f810eac421c4c02c68447eb5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4134/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026448019385225216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4134,"pr_state":"closed","pr_sha":"9fdaef42a0663c4aa5d5cd32e40a46627c1d79fb","final_merged_sha":"9fdaef42a0663c4aa5d5cd32e40a46627c1d79fb","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-25T00:03:51Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"3c7c0834d6979882326e90ac87fccb2ffec7e15bb2963df121276dea95f50a41","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4134/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026448019385225216","occurred_at":"2026-02-25T00:03:51Z","failed_at":"provision","signature_id":"c0e73729f65c1bddce0d205afd5219d39a2909192e457ed4b8bdc151c1b7e673","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-02-25T00:54:51.286Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e secretSyncController.image.registry \u003cnil\u003e}:{\u003cnil\u003e secretSyncController.image.repository \u003cnil\u003e}@{\u003cnil\u003e secretSyncController.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T00:54:52.638Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry registry.k8s.io from arohcpdev-global KV.\\nERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8c9fc918c1d28a5fd2dc0ad245350a62ec83b0c2f810eac421c4c02c68447eb5","failure_pattern":"ERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4","search_query":"ERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8c9fc918c1d28a5fd2dc0ad245350a62ec83b0c2f810eac421c4c02c68447eb5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026447392777179136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4134,"pr_state":"closed","pr_sha":"9fdaef42a0663c4aa5d5cd32e40a46627c1d79fb","final_merged_sha":"9fdaef42a0663c4aa5d5cd32e40a46627c1d79fb","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-02-25T00:01:06Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"9761e931829a831aab1d4c6551878a815b711f176c82be550ec3992504c9165a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2026447392777179136","occurred_at":"2026-02-25T00:01:06Z","failed_at":"provision","signature_id":"cdf8d8e2d103a4b3ab1c2a6232912f6a711187ae189bab007a2056b3d88cbc25","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Agent/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-02-25T00:59:54.329Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e maestro.image.registry \u003cnil\u003e}:{\u003cnil\u003e maestro.image.repository \u003cnil\u003e}@{\u003cnil\u003e maestro.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-02-25T00:59:56.175Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org registry.build06.ci.openshift.org\\nFetch pull secret for source registry quay.io from arohcpdev-global KV.\\nERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8c9fc918c1d28a5fd2dc0ad245350a62ec83b0c2f810eac421c4c02c68447eb5","failure_pattern":"ERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4","search_query":"ERROR: Invalid base64-encoded string: number of data characters (1325) cannot be 1 more than a multiple of 4"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["8c9fc918c1d28a5fd2dc0ad245350a62ec83b0c2f810eac421c4c02c68447eb5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}