{"meta":{"date":"2026-03-14","timezone":"UTC","generated_at":"2026-11-01T00:08:45Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":19,"failed_runs":3,"runs_with_occurrences":3,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":16,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4384/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032947491288125440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4384,"pr_state":"closed","pr_sha":"76945bffc05e0be9c6747a6366cbd4377839d85e","final_merged_sha":"76945bffc05e0be9c6747a6366cbd4377839d85e","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-14T22:30:26Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4463/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032932424396050432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4463,"pr_state":"open","pr_sha":"8bf53059836d7c658117add712660a10ebbab48b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-14T21:30:34Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4462/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032842693184327680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4462,"pr_state":"closed","pr_sha":"cb53a8c398910ebc8c0baf92843e0a327b766654","final_merged_sha":"cb53a8c398910ebc8c0baf92843e0a327b766654","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-14T15:34:01Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032809337855938560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4350,"pr_state":"closed","pr_sha":"c117b0a88ebb40e9ee7f4bec2c1680e806014b69","final_merged_sha":"c117b0a88ebb40e9ee7f4bec2c1680e806014b69","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-14T13:21:28Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032808637956624384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4350,"pr_state":"closed","pr_sha":"c117b0a88ebb40e9ee7f4bec2c1680e806014b69","final_merged_sha":"c117b0a88ebb40e9ee7f4bec2c1680e806014b69","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-14T13:18:41Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4281/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032803916978065408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4281,"pr_state":"open","pr_sha":"f55caa5641a1457b14fd2f647226597c888b9484","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-14T12:59:56Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4434/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032778782510682112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4434,"pr_state":"closed","pr_sha":"0225b04f47043dc7c700ad7ea2f1adb6e0288533","final_merged_sha":"0225b04f47043dc7c700ad7ea2f1adb6e0288533","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-14T11:20:03Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032778090857369600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4434,"pr_state":"closed","pr_sha":"0225b04f47043dc7c700ad7ea2f1adb6e0288533","final_merged_sha":"0225b04f47043dc7c700ad7ea2f1adb6e0288533","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-14T11:17:18Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4432/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032771655327027200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4432,"pr_state":"closed","pr_sha":"b1ab51614479a0dd3e1236b827fc4ff1f09851f1","final_merged_sha":"b1ab51614479a0dd3e1236b827fc4ff1f09851f1","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-14T10:51:44Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4462/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032768940425351168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4462,"pr_state":"closed","pr_sha":"cb53a8c398910ebc8c0baf92843e0a327b766654","final_merged_sha":"cb53a8c398910ebc8c0baf92843e0a327b766654","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-14T10:40:57Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4350/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032767516949876736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4350,"pr_state":"closed","pr_sha":"c117b0a88ebb40e9ee7f4bec2c1680e806014b69","final_merged_sha":"c117b0a88ebb40e9ee7f4bec2c1680e806014b69","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-14T10:35:17Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4432/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032751938667286528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4432,"pr_state":"closed","pr_sha":"b1ab51614479a0dd3e1236b827fc4ff1f09851f1","final_merged_sha":"b1ab51614479a0dd3e1236b827fc4ff1f09851f1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-14T09:33:23Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ae07086d39c44d12d1eb220e89832c7db50d6d7395052d6cbc361f27f8818a69","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4432/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032751938667286528","occurred_at":"2026-03-14T09:33:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032751275073867776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4443,"pr_state":"closed","pr_sha":"53a705301c2acfc2a39dcba371fa99ace1ffd433","final_merged_sha":"53a705301c2acfc2a39dcba371fa99ace1ffd433","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-14T09:30:40Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4447/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032743700571885568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4447,"pr_state":"closed","pr_sha":"3ef05c4706062e1df26f34038382392c0d192b35","final_merged_sha":"3ef05c4706062e1df26f34038382392c0d192b35","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-14T08:59:38Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4447/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032722501196320768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4447,"pr_state":"closed","pr_sha":"3ef05c4706062e1df26f34038382392c0d192b35","final_merged_sha":"3ef05c4706062e1df26f34038382392c0d192b35","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-14T07:36:25Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b2b1f1ebcbcc74c15595c3f8542f22ae05264c6430b268b52c95f9d10cdeb1d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4447/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032722501196320768","occurred_at":"2026-03-14T07:36:25Z","failed_at":"provision","signature_id":"7624756560b407085d2513386b5bd80f01cdfdbc8b740a68c1c1c2953aa78945","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.AdminAPI/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-03-14T08:14:00.273Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e adminApi.image.registry \u003cnil\u003e}:{\u003cnil\u003e adminApi.image.repository \u003cnil\u003e}@{\u003cnil\u003e adminApi.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-14T08:14:45.685Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.AdminAPI resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org registry.build02.ci.openshift.org\\nSetting up registry authentication for CI source registry.\\ninfo: Using registry public hostname registry.build02.ci.openshift.org\\nSaved credentials for registry.build02.ci.openshift.org into /tmp/tmp.gB2LpZjzpB/containers/auth.json\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image registry.build02.ci.openshift.org/ci-op-ksw6xgy9/pipeline@sha256:e0e04b2170b0b492723bcbd23f2d5023ec91c7bfdf45150f792da90c84af718f to arohcpsvcdev.azurecr.io/ci-op-ksw6xgy9/pipeline:e0e04b2170b0b492723bcbd23f2d5023ec91c7bfdf45150f792da90c84af718f.\\nThe image will still be available under it's original digest sha256:e0e04b2170b0b492723bcbd23f2d5023ec91c7bfdf45150f792da90c84af718f in the target registry.\\nError response from registry: recognizable error message not found: PUT \\\"https://arohcpsvcdev.azurecr.io/v2/ci-op-ksw6xgy9/pipeline/blobs/uploads/9df0bd65-a39f-4377-8874-bb2e6b531a8e?_nouploadcache=false\u0026_state=nbImm9jwgAShWR_yFdBZhSe0NLeAIa2-SEx25L0qTlF7Ik5hbWUiOiJjaS1vcC1rc3c2eGd5OS9waXBlbGluZSIsIlVVSUQiOiI5ZGYwYmQ2NS1hMzlmLTQzNzctODg3NC1iYjJlNmI1MzFhOGUiLCJPZmZzZXQiOjAsIlN0YXJ0ZWRBdCI6IjIwMjYtMDMtMTRUMDg6MTQ6MDkuMzYxODI5MTk5WiJ9\u0026digest=sha256%3Aefac5040dde15d780c2abdd649899044b184cc6bf93166c1ad6b345a6e529a69\\\": response status code 502: Bad Gateway\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"136ce069c8492df46aa49e1d77285d5c5962f77589ba0fe5e41d5e51bfa7ba59","failure_pattern":"Error response from registry: recognizable error message not found: PUT \"\u003curl\u003e\": response status code 502: Bad Gateway","search_query":"Error response from registry: recognizable error message not found: PUT \\\"https://arohcpsvcdev.azurecr.io/v2/ci-op-ksw6xgy9/pipeline/blobs/uploads/9df0bd65-a39f-4377-8874-bb2e6b531a8e?_nouploadcache=false\u0026_state=nbImm9jwgAShWR_yFdBZhSe0NLeAIa2-SEx25L0qTlF7Ik5hbWUiOiJjaS1vcC1rc3c2eGd5OS9waXBlbGluZSIsIlVVSUQiOiI5ZGYwYmQ2NS1hMzlmLTQzNzctODg3NC1iYjJlNmI1MzFhOGUiLCJPZmZzZXQiOjAsIlN0YXJ0ZWRBdCI6IjIwMjYtMDMtMTRUMDg6MTQ6MDkuMzYxODI5MTk5WiJ9\u0026digest=sha256%3Aefac5040dde15d780c2abdd649899044b184cc6bf93166c1ad6b345a6e529a69\\\": response status code 502: Bad Gateway"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["136ce069c8492df46aa49e1d77285d5c5962f77589ba0fe5e41d5e51bfa7ba59"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4461/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032656538438471680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4461,"pr_state":"closed","pr_sha":"b806d7769e69345698f74d7a23f602c5bb2a17f2","final_merged_sha":"9baa329552156f993c1fc5b382b78423cd754fef","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-14T03:14:18Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4455/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032620331960635392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4455,"pr_state":"closed","pr_sha":"a484243a8fad178e081f8e0f89d8db8e300a58a1","final_merged_sha":"7b9532af16d82b9a9786a1179527a9bf0cc955bb","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-14T00:50:26Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4381/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032620208488714240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4381,"pr_state":"closed","pr_sha":"0cc8e93d1fbc7dad972a20ddb7a146ef2939b3c9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-03-14T00:49:56Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b0313b0782504bc412172bcea619cacdb0f1bb515a1d004883296814f878529e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4381/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032620208488714240","occurred_at":"2026-03-14T00:49:56Z","failed_at":"provision","signature_id":"0774ddd169db58345713fd2bb812fc265dd06d5de1818adcefa324a2e6a9e3c7","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SecretSyncController/global/mirror-image","test_suite":"step graph","failure_text":"time=2026-03-14T01:20:23.888Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e secretSyncController.image.registry \u003cnil\u003e}:{\u003cnil\u003e secretSyncController.image.repository \u003cnil\u003e}@{\u003cnil\u003e secretSyncController.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-14T01:20:24.337Z level=INFO msg=\"Fetching pull secret from Key Vault\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image vault=arohcpdev-global secret=component-sync-pull-secret\ntime=2026-03-14T01:20:24.824Z level=INFO msg=\"No credentials found in pull secret for source registry, using anonymous access\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image registry=registry.k8s.io\ntime=2026-03-14T01:20:24.824Z level=INFO msg=\"Logging into target ACR\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image acr=arohcpsvcdev\ntime=2026-03-14T01:20:26.794Z level=INFO msg=\"Mirroring image\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image source=registry.k8s.io/secrets-store-sync/controller@sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5 target=arohcpsvcdev.azurecr.io/secrets-store-sync/controller:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5\ntime=2026-03-14T01:20:27.139Z level=INFO msg=\"Image copy failed, retrying\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image attempt=1 error=\"failed to copy image: failed to perform \\\"Exists\\\" on destination: HEAD \\\"https://arohcpsvcdev.azurecr.io/v2/secrets-store-sync/controller/manifests/sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5\\\": GET \\\"https://arohcpsvcdev.azurecr.io/oauth2/token?scope=repository%3Asecrets-store-sync%2Fcontroller%3Apull\u0026service=arohcpsvcdev.azurecr.io\\\": response status code 401: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: ee732d6b-7f98-436a-93f0-81cd7c963d31\"\ntime=2026-03-14T01:20:57.231Z level=INFO msg=\"Image copy failed, retrying\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image attempt=2 error=\"failed to copy image: failed to perform \\\"Exists\\\" on destination: HEAD \\\"https://arohcpsvcdev.azurecr.io/v2/secrets-store-sync/controller/manifests/sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5\\\": GET \\\"https://arohcpsvcdev.azurecr.io/oauth2/token?scope=repository%3Asecrets-store-sync%2Fcontroller%3Apull\u0026service=arohcpsvcdev.azurecr.io\\\": response status code 401: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 30e9cac5-42c5-4277-9a75-afb2b079d046\"\ntime=2026-03-14T01:21:27.334Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image step=mirror-image error=\"error running Image Mirror Step, failed to mirror image after 3 attempts: failed to copy image: failed to perform \\\"Exists\\\" on destination: HEAD \\\"https://arohcpsvcdev.azurecr.io/v2/secrets-store-sync/controller/manifests/sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5\\\": GET \\\"https://arohcpsvcdev.azurecr.io/oauth2/token?scope=repository%3Asecrets-store-sync%2Fcontroller%3Apull\u0026service=arohcpsvcdev.azurecr.io\\\": response status code 401: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 952ede4c-929d-46f8-a3e7-3d00c7512a51\"\ntime=2026-03-14T01:22:27.335Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e secretSyncController.image.registry \u003cnil\u003e}:{\u003cnil\u003e secretSyncController.image.repository \u003cnil\u003e}@{\u003cnil\u003e secretSyncController.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-14T01:22:27.732Z level=INFO msg=\"Fetching pull secret from Key Vault\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image vault=arohcpdev-global secret=component-sync-pull-secret\ntime=2026-03-14T01:22:28.109Z level=INFO msg=\"No credentials found in pull secret for source registry, using anonymous access\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image registry=registry.k8s.io\ntime=2026-03-14T01:22:28.109Z level=INFO msg=\"Logging into target ACR\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image acr=arohcpsvcdev\ntime=2026-03-14T01:22:30.153Z level=INFO msg=\"Mirroring image\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image source=registry.k8s.io/secrets-store-sync/controller@sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5 target=arohcpsvcdev.azurecr.io/secrets-store-sync/controller:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5\ntime=2026-03-14T01:22:30.256Z level=INFO msg=\"Image copy failed, retrying\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image attempt=1 error=\"failed to copy image: failed to perform \\\"Exists\\\" on destination: HEAD \\\"https://arohcpsvcdev.azurecr.io/v2/secrets-store-sync/controller/manifests/sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5\\\": GET \\\"https://arohcpsvcdev.azurecr.io/oauth2/token?scope=repository%3Asecrets-store-sync%2Fcontroller%3Apull\u0026service=arohcpsvcdev.azurecr.io\\\": response status code 401: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 09be7b06-64ee-4a12-85c7-dd9640f4c9ad\"\ntime=2026-03-14T01:23:00.354Z level=INFO msg=\"Image copy failed, retrying\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image attempt=2 error=\"failed to copy image: failed to perform \\\"Exists\\\" on destination: HEAD \\\"https://arohcpsvcdev.azurecr.io/v2/secrets-store-sync/controller/manifests/sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5\\\": GET \\\"https://arohcpsvcdev.azurecr.io/oauth2/token?scope=repository%3Asecrets-store-sync%2Fcontroller%3Apull\u0026service=arohcpsvcdev.azurecr.io\\\": response status code 401: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 3cbef2bb-7ae5-44e8-813c-967947b3bf8a\"\ntime=2026-03-14T01:23:30.466Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image step=mirror-image error=\"error running Image Mirror Step, failed to mirror image after 3 attempts: failed to copy image: failed to perform \\\"Exists\\\" on destination: HEAD \\\"https://arohcpsvcdev.azurecr.io/v2/secrets-store-sync/controller/manifests/sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5\\\": GET \\\"https://arohcpsvcdev.azurecr.io/oauth2/token?scope=repository%3Asecrets-store-sync%2Fcontroller%3Apull\u0026service=arohcpsvcdev.azurecr.io\\\": response status code 401: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 6b662981-d0b1-408e-ad26-56a22700bb80\"\ntime=2026-03-14T01:24:30.466Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e secretSyncController.image.registry \u003cnil\u003e}:{\u003cnil\u003e secretSyncController.image.repository \u003cnil\u003e}@{\u003cnil\u003e secretSyncController.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-14T01:24:30.891Z level=INFO msg=\"Fetching pull secret from Key Vault\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image vault=arohcpdev-global secret=component-sync-pull-secret\ntime=2026-03-14T01:24:31.267Z level=INFO msg=\"No credentials found in pull secret for source registry, using anonymous access\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image registry=registry.k8s.io\ntime=2026-03-14T01:24:31.267Z level=INFO msg=\"Logging into target ACR\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image acr=arohcpsvcdev\ntime=2026-03-14T01:24:33.081Z level=INFO msg=\"Mirroring image\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image source=registry.k8s.io/secrets-store-sync/controller@sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5 target=arohcpsvcdev.azurecr.io/secrets-store-sync/controller:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5\ntime=2026-03-14T01:24:33.185Z level=INFO msg=\"Image copy failed, retrying\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image attempt=1 error=\"failed to copy image: failed to perform \\\"Exists\\\" on destination: HEAD \\\"https://arohcpsvcdev.azurecr.io/v2/secrets-store-sync/controller/manifests/sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5\\\": GET \\\"https://arohcpsvcdev.azurecr.io/oauth2/token?scope=repository%3Asecrets-store-sync%2Fcontroller%3Apull\u0026service=arohcpsvcdev.azurecr.io\\\": response status code 401: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 83189509-63f7-44ff-aa4e-504463e119e6\"\ntime=2026-03-14T01:25:03.293Z level=INFO msg=\"Image copy failed, retrying\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image attempt=2 error=\"failed to copy image: failed to perform \\\"Exists\\\" on destination: HEAD \\\"https://arohcpsvcdev.azurecr.io/v2/secrets-store-sync/controller/manifests/sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5\\\": GET \\\"https://arohcpsvcdev.azurecr.io/oauth2/token?scope=repository%3Asecrets-store-sync%2Fcontroller%3Apull\u0026service=arohcpsvcdev.azurecr.io\\\": response status code 401: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 278add4a-a8db-4827-bad3-8af288376a2a\"\ntime=2026-03-14T01:25:33.387Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image step=mirror-image error=\"error running Image Mirror Step, failed to mirror image after 3 attempts: failed to copy image: failed to perform \\\"Exists\\\" on destination: HEAD \\\"https://arohcpsvcdev.azurecr.io/v2/secrets-store-sync/controller/manifests/sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5\\\": GET \\\"https://arohcpsvcdev.azurecr.io/oauth2/token?scope=repository%3Asecrets-store-sync%2Fcontroller%3Apull\u0026service=arohcpsvcdev.azurecr.io\\\": response status code 401: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: c1ad9bca-61e2-45ad-937c-5b703b3127db\"\ntime=2026-03-14T01:26:33.388Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e secretSyncController.image.registry \u003cnil\u003e}:{\u003cnil\u003e secretSyncController.image.repository \u003cnil\u003e}@{\u003cnil\u003e secretSyncController.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-14T01:26:33.788Z level=INFO msg=\"Fetching pull secret from Key Vault\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image vault=arohcpdev-global secret=component-sync-pull-secret\ntime=2026-03-14T01:26:34.148Z level=INFO msg=\"No credentials found in pull secret for source registry, using anonymous access\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image registry=registry.k8s.io\ntime=2026-03-14T01:26:34.148Z level=INFO msg=\"Logging into target ACR\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image acr=arohcpsvcdev\ntime=2026-03-14T01:26:36.073Z level=INFO msg=\"Mirroring image\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image source=registry.k8s.io/secrets-store-sync/controller@sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5 target=arohcpsvcdev.azurecr.io/secrets-store-sync/controller:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5\ntime=2026-03-14T01:26:36.173Z level=INFO msg=\"Image copy failed, retrying\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image attempt=1 error=\"failed to copy image: failed to perform \\\"Exists\\\" on destination: HEAD \\\"https://arohcpsvcdev.azurecr.io/v2/secrets-store-sync/controller/manifests/sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5\\\": GET \\\"https://arohcpsvcdev.azurecr.io/oauth2/token?scope=repository%3Asecrets-store-sync%2Fcontroller%3Apull\u0026service=arohcpsvcdev.azurecr.io\\\": response status code 401: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 08964b2e-2c71-4950-b2f4-5fa65851a25b\"\ntime=2026-03-14T01:27:06.277Z level=INFO msg=\"Image copy failed, retrying\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image attempt=2 error=\"failed to copy image: failed to perform \\\"Exists\\\" on destination: HEAD \\\"https://arohcpsvcdev.azurecr.io/v2/secrets-store-sync/controller/manifests/sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5\\\": GET \\\"https://arohcpsvcdev.azurecr.io/oauth2/token?scope=repository%3Asecrets-store-sync%2Fcontroller%3Apull\u0026service=arohcpsvcdev.azurecr.io\\\": response status code 401: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: c606fc67-4c14-4950-876e-611791e70556\"\ntime=2026-03-14T01:27:36.401Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image step=mirror-image error=\"error running Image Mirror Step, failed to mirror image after 3 attempts: failed to copy image: failed to perform \\\"Exists\\\" on destination: HEAD \\\"https://arohcpsvcdev.azurecr.io/v2/secrets-store-sync/controller/manifests/sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5\\\": GET \\\"https://arohcpsvcdev.azurecr.io/oauth2/token?scope=repository%3Asecrets-store-sync%2Fcontroller%3Apull\u0026service=arohcpsvcdev.azurecr.io\\\": response status code 401: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 32e3f887-26fc-4b19-89ce-47e411351541\"\ntime=2026-03-14T01:28:36.401Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e secretSyncController.image.registry \u003cnil\u003e}:{\u003cnil\u003e secretSyncController.image.repository \u003cnil\u003e}@{\u003cnil\u003e secretSyncController.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-03-14T01:28:36.825Z level=INFO msg=\"Fetching pull secret from Key Vault\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image vault=arohcpdev-global secret=component-sync-pull-secret\ntime=2026-03-14T01:28:37.216Z level=INFO msg=\"No credentials found in pull secret for source registry, using anonymous access\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image registry=registry.k8s.io\ntime=2026-03-14T01:28:37.216Z level=INFO msg=\"Logging into target ACR\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image acr=arohcpsvcdev\ntime=2026-03-14T01:28:39.172Z level=INFO msg=\"Mirroring image\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image source=registry.k8s.io/secrets-store-sync/controller@sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5 target=arohcpsvcdev.azurecr.io/secrets-store-sync/controller:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5\ntime=2026-03-14T01:28:39.284Z level=INFO msg=\"Image copy failed, retrying\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image attempt=1 error=\"failed to copy image: failed to perform \\\"Exists\\\" on destination: HEAD \\\"https://arohcpsvcdev.azurecr.io/v2/secrets-store-sync/controller/manifests/sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5\\\": GET \\\"https://arohcpsvcdev.azurecr.io/oauth2/token?scope=repository%3Asecrets-store-sync%2Fcontroller%3Apull\u0026service=arohcpsvcdev.azurecr.io\\\": response status code 401: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 1a85d615-9114-49e6-85cd-7c95709dca5e\"\ntime=2026-03-14T01:29:09.388Z level=INFO msg=\"Image copy failed, retrying\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image attempt=2 error=\"failed to copy image: failed to perform \\\"Exists\\\" on destination: HEAD \\\"https://arohcpsvcdev.azurecr.io/v2/secrets-store-sync/controller/manifests/sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5\\\": GET \\\"https://arohcpsvcdev.azurecr.io/oauth2/token?scope=repository%3Asecrets-store-sync%2Fcontroller%3Apull\u0026service=arohcpsvcdev.azurecr.io\\\": response status code 401: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: e8a480e2-1fc8-4bc5-8316-2771b285d04e\"\ntime=2026-03-14T01:29:39.492Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image step=mirror-image error=\"error running Image Mirror Step, failed to mirror image after 3 attempts: failed to copy image: failed to perform \\\"Exists\\\" on destination: HEAD \\\"https://arohcpsvcdev.azurecr.io/v2/secrets-store-sync/controller/manifests/sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5\\\": GET \\\"https://arohcpsvcdev.azurecr.io/oauth2/token?scope=repository%3Asecrets-store-sync%2Fcontroller%3Apull\u0026service=arohcpsvcdev.azurecr.io\\\": response status code 401: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 406bc5e7-5caf-4f3e-a1f7-2d24e679eb5e\"\ntime=2026-03-14T01:30:39.493Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SecretSyncController resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to mirror image after 3 attempts: failed to copy image: failed to perform \\\"Exists\\\" on destination: HEAD \\\"https://arohcpsvcdev.azurecr.io/v2/secrets-store-sync/controller/manifests/sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5\\\": GET \\\"https://arohcpsvcdev.azurecr.io/oauth2/token?scope=repository%3Asecrets-store-sync%2Fcontroller%3Apull\u0026service=arohcpsvcdev.azurecr.io\\\": response status code 401: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 406bc5e7-5caf-4f3e-a1f7-2d24e679eb5e\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9493e33e54b2f0c82c9e14169bd67adaa49ef562592f8191990c54c96280e81b","failure_pattern":"error running Image Mirror Step, failed to mirror image after 3 attempts: failed to copy image: failed to perform \"Exists\" on destination: HEAD \"\u003curl\u003e\": GET \"\u003curl\u003e\": response status code 401: unauthorized: authentication required, visit \u003curl\u003e for more","search_query":"error running Image Mirror Step, failed to mirror image after 3 attempts: failed to copy image: failed to perform \\\"Exists\\\" on destination: HEAD \\\"https://arohcpsvcdev.azurecr.io/v2/secrets-store-sync/controller/manifests/sha256:15768ff940ba7744d9994f8abde9ceb3ec3c5c6a024cb1256feb4d1b17c6b9f5\\\": GET \\\"https://arohcpsvcdev.azurecr.io/oauth2/token?scope=repository%3Asecrets-store-sync%2Fcontroller%3Apull\u0026service=arohcpsvcdev.azurecr.io\\\": response status code 401: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 406bc5e7-5caf-4f3e-a1f7-2d24e679eb5e"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9493e33e54b2f0c82c9e14169bd67adaa49ef562592f8191990c54c96280e81b"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/pr-logs/pull/Azure_ARO-HCP/4457/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2032616199036080128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":4457,"pr_state":"open","pr_sha":"c6d34c6bc54bf82f40b7a2b9e91f43b977c11e31","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-03-14T00:34:00Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}