{"meta":{"start_date":"2026-03-30","end_date":"2026-03-30","start_at":"2026-03-30T00:00:00Z","end_at":"2026-03-31T00:00:00Z","timezone":"UTC","generated_at":"2026-08-27T14:41:38Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":4,"failed_runs":3,"raw_occurrences":11,"matched_occurrences":11,"runs_affected":3},"rows":[{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dds6fm, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dds6fm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dds6fm, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":5,"runs_affected":2,"failed_runs":2,"run_impact_percent":50,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,0,5],"trend_range":"2026-03-24..2026-03-30 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-02","2026-03-16","2026-03-23"],"prior_runs_affected":12,"prior_last_seen_at":"2026-03-26T19:56:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:141]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f88120\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-6npjkh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-6npjkh, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-6npjkh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-6npjkh, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001038e50\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-6npjkh\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:141]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f88120\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-6npjkh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-6npjkh, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-6npjkh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-6npjkh, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001038e50\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-6npjkh\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:135]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001380270\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-5qfzqk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-5qfzqk, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-5qfzqk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-5qfzqk, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010e41b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-5qfzqk\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:135]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001380270\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-5qfzqk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-5qfzqk, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-5qfzqk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-5qfzqk, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010e41b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-5qfzqk\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:135]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0006da630\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-45j7m6\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-45j7m6, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-45j7m6\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-45j7m6, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f9d270\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-45j7m6\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:135]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0006da630\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-45j7m6\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-45j7m6, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-45j7m6\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-45j7m6, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f9d270\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-45j7m6\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038765373188214784","occurred_at":"2026-03-30T23:48:38Z","signature_id":"450fc46eb897ae03418c2fe9eac2322e9500a3838e192b479c4a2a2393594121","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038765373188214784","occurred_at":"2026-03-30T23:48:38Z","signature_id":"cfdc42f7f5e1bb71834c753a6705995745ffdeec1d0a9fde66b9179df2367e2d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038721439078551552","occurred_at":"2026-03-30T20:54:03Z","signature_id":"1a813e68e6ddf8f8546a168c340941d04d5b6ea6a1c70f99a114e33c5613f7df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038721439078551552","occurred_at":"2026-03-30T20:54:03Z","signature_id":"dde9a81a90c6df6068b9056997e34d925cd5b8143c637373a77e989a1b24dd84","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038721439078551552","occurred_at":"2026-03-30T20:54:03Z","signature_id":"fbefb27382b80d49faff96db2dd21ef9d35a0bc5238a6eeb1600aac3e05ac1c9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"e17b72ff49490f6e1940fa5deb5057f7012ebd4b2cee2f2d2e8b8d47e6784bb9","failure_pattern":"ERROR CODE: LockContention; detail message Failed to acquire lock: timed out; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LockContention","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":5,"after_last_push_count":9,"trend_counts":[0,0,0,0,0,0,5],"trend_range":"2026-03-24..2026-03-30 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-02","2026-03-09"],"prior_runs_affected":5,"prior_last_seen_at":"2026-03-13T06:56:30Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e760c0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-pkkxdw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: LockContention\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LockContention\",\n \"message\": \"Failed to acquire lock: timed out\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-pkkxdw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"LockContention\\\",\\n \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000dce1e0\u003e{\n ErrorCode: \"LockContention\",\n StatusCode: 503,\n RawResponse: {\n Status: \"503 Service Unavailable\",\n StatusCode: 503,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Content-Length\": [\"186\"],\n \"X-Ms-Error-Code\": [\"LockContention\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Pragma\": [\"no-cache\"],\n \"Retry-After\": [\"10\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"37ef49af-09e9-40ee-bfca-efc082a80f64\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260330T021725Z:37ef49af-09e9-40ee-bfca-efc082a80f64\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Content-Type\": [\"application/json\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Request-Id\": [\n \"e0a372c3-fcdc-4ce9-bf08-e2fce9bc2797\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"10011\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/3345b2b1-e23f-4ce2-8dab-587f9642a08d\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: 93A8B2C021534273A8B574371656B5C5 Ref B: CH1AA2020620025 Ref C: 2026-03-30T02:17:14Z\",\n ],\n \"Date\": [\n \"Mon, 30 Mar 2026 02:17:24 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000e76000\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 76, 111, 99, 107, 67, 111, 110, 116, 101, 110, 116, 105, 111, 110, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 32, 116, 111, 32, 97, 99, 113, 117, 105, 114, 101, 32, 108, 111, 99, 107, 58, 32, 116, 105, 109, 101, 100, 32, 111, 117, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 57, 51, 57, 57, 50, 56, 49, 45, 48, 48, 97, 50, 45, 52, 98, 51, 57, 45, 98, 98, 51, 100, 45, 98, 50, 54, 52, 53, 98, 98, 98, 100, 98, 57, 51, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e760c0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-pkkxdw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: LockContention\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LockContention\",\n \"message\": \"Failed to acquire lock: timed out\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-pkkxdw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"LockContention\\\",\\n \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000dce1e0\u003e{\n ErrorCode: \"LockContention\",\n StatusCode: 503,\n RawResponse: {\n Status: \"503 Service Unavailable\",\n StatusCode: 503,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Content-Length\": [\"186\"],\n \"X-Ms-Error-Code\": [\"LockContention\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Pragma\": [\"no-cache\"],\n \"Retry-After\": [\"10\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"37ef49af-09e9-40ee-bfca-efc082a80f64\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260330T021725Z:37ef49af-09e9-40ee-bfca-efc082a80f64\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Content-Type\": [\"application/json\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Request-Id\": [\n \"e0a372c3-fcdc-4ce9-bf08-e2fce9bc2797\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"10011\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/3345b2b1-e23f-4ce2-8dab-587f9642a08d\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: 93A8B2C021534273A8B574371656B5C5 Ref B: CH1AA2020620025 Ref C: 2026-03-30T02:17:14Z\",\n ],\n \"Date\": [\n \"Mon, 30 Mar 2026 02:17:24 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000e76000\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 76, 111, 99, 107, 67, 111, 110, 116, 101, 110, 116, 105, 111, 110, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 32, 116, 111, 32, 97, 99, 113, 117, 105, 114, 101, 32, 108, 111, 99, 107, 58, 32, 116, 105, 109, 101, 100, 32, 111, 117, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 57, 51, 57, 57, 50, 56, 49, 45, 48, 48, 97, 50, 45, 52, 98, 51, 57, 45, 98, 98, 51, 100, 45, 98, 50, 54, 52, 53, 98, 98, 98, 100, 98, 57, 51, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:106]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000dce100\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-6smz57/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: LockContention\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LockContention\",\n \"message\": \"Failed to acquire lock: timed out\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-6smz57/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"LockContention\\\",\\n \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000d7a2d0\u003e{\n ErrorCode: \"LockContention\",\n StatusCode: 503,\n RawResponse: {\n Status: \"503 Service Unavailable\",\n StatusCode: 503,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260330T021728Z:f051cbc0-873c-44c2-8914-5e59d922dd86\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"Retry-After\": [\"10\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/3ff18aa5-fc8a-478d-a260-ce0acf33be72\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"f051cbc0-873c-44c2-8914-5e59d922dd86\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Content-Length\": [\"186\"],\n \"Expires\": [\"-1\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: E71DB15DBD354DA688879002947B9080 Ref B: CHI261140506040 Ref C: 2026-03-30T02:17:18Z\",\n ],\n \"Date\": [\n \"Mon, 30 Mar 2026 02:17:27 GMT\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Error-Code\": [\"LockContention\"],\n \"X-Ms-Request-Id\": [\n \"180208ba-d8e7-4063-898e-2652baca60d0\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"10014\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000dce040\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 76, 111, 99, 107, 67, 111, 110, 116, 101, 110, 116, 105, 111, 110, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 32, 116, 111, 32, 97, 99, 113, 117, 105, 114, 101, 32, 108, 111, 99, 107, 58, 32, 116, 105, 109, 101, 100, 32, 111, 117, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 57, 51, 57, 57, 50, 56, 49, 45, 48, 48, 97, 50, 45, 52, 98, 51, 57, 45, 98, 98, 51, 100, 45, 98, 50, 54, 52, 53, 98, 98, 98, 100, 98, 57, 51, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n Con...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:106]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000dce100\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-6smz57/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: LockContention\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LockContention\",\n \"message\": \"Failed to acquire lock: timed out\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-6smz57/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"LockContention\\\",\\n \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000d7a2d0\u003e{\n ErrorCode: \"LockContention\",\n StatusCode: 503,\n RawResponse: {\n Status: \"503 Service Unavailable\",\n StatusCode: 503,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260330T021728Z:f051cbc0-873c-44c2-8914-5e59d922dd86\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"Retry-After\": [\"10\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/3ff18aa5-fc8a-478d-a260-ce0acf33be72\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"f051cbc0-873c-44c2-8914-5e59d922dd86\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Content-Length\": [\"186\"],\n \"Expires\": [\"-1\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: E71DB15DBD354DA688879002947B9080 Ref B: CHI261140506040 Ref C: 2026-03-30T02:17:18Z\",\n ],\n \"Date\": [\n \"Mon, 30 Mar 2026 02:17:27 GMT\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Error-Code\": [\"LockContention\"],\n \"X-Ms-Request-Id\": [\n \"180208ba-d8e7-4063-898e-2652baca60d0\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"10014\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000dce040\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 76, 111, 99, 107, 67, 111, 110, 116, 101, 110, 116, 105, 111, 110, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 32, 116, 111, 32, 97, 99, 113, 117, 105, 114, 101, 32, 108, 111, 99, 107, 58, 32, 116, 105, 109, 101, 100, 32, 111, 117, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 57, 51, 57, 57, 50, 56, 49, 45, 48, 48, 97, 50, 45, 52, 98, 51, 57, 45, 98, 98, 51, 100, 45, 98, 50, 54, 52, 53, 98, 98, 98, 100, 98, 57, 51, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n Con...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:94]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0009a2240\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-z7kqxx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: LockContention\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LockContention\",\n \"message\": \"Failed to acquire lock: timed out\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-z7kqxx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"LockContention\\\",\\n \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001452210\u003e{\n ErrorCode: \"LockContention\",\n StatusCode: 503,\n RawResponse: {\n Status: \"503 Service Unavailable\",\n StatusCode: 503,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Retry-After\": [\"10\"],\n \"X-Ms-Request-Id\": [\n \"bad8f67e-95d5-4627-9ade-5f4404fb64c1\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Date\": [\n \"Mon, 30 Mar 2026 02:17:34 GMT\",\n ],\n \"Content-Length\": [\"186\"],\n \"Content-Type\": [\"application/json\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\"LockContention\"],\n \"X-Envoy-Upstream-Service-Time\": [\"10022\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/9626a40b-40ac-4fb0-8b06-04ed9b8c866f\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260330T021735Z:d50c9684-e91b-44fe-9754-322d764cfb29\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"d50c9684-e91b-44fe-9754-322d764cfb29\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 25490A5A563241E3AB9615419CA0144E Ref B: CHI261140503029 Ref C: 2026-03-30T02:17:24Z\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0009a20e0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 76, 111, 99, 107, 67, 111, 110, 116, 101, 110, 116, 105, 111, 110, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 32, 116, 111, 32, 97, 99, 113, 117, 105, 114, 101, 32, 108, 111, 99, 107, 58, 32, 116, 105, 109, 101, 100, 32, 111, 117, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 57, 51, 57, 57, 50, 56, 49, 45, 48, 48, 97, 50, 45, 52, 98, 51, 57, 45, 98, 98, 51, 100, 45, 98, 50, 54, 52, 53, 98, 98, 98, 100, 98, 57, 51, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n Conte...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:94]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0009a2240\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-z7kqxx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: LockContention\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LockContention\",\n \"message\": \"Failed to acquire lock: timed out\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-z7kqxx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"LockContention\\\",\\n \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001452210\u003e{\n ErrorCode: \"LockContention\",\n StatusCode: 503,\n RawResponse: {\n Status: \"503 Service Unavailable\",\n StatusCode: 503,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Retry-After\": [\"10\"],\n \"X-Ms-Request-Id\": [\n \"bad8f67e-95d5-4627-9ade-5f4404fb64c1\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Date\": [\n \"Mon, 30 Mar 2026 02:17:34 GMT\",\n ],\n \"Content-Length\": [\"186\"],\n \"Content-Type\": [\"application/json\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\"LockContention\"],\n \"X-Envoy-Upstream-Service-Time\": [\"10022\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/9626a40b-40ac-4fb0-8b06-04ed9b8c866f\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260330T021735Z:d50c9684-e91b-44fe-9754-322d764cfb29\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"d50c9684-e91b-44fe-9754-322d764cfb29\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 25490A5A563241E3AB9615419CA0144E Ref B: CHI261140503029 Ref C: 2026-03-30T02:17:24Z\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0009a20e0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 76, 111, 99, 107, 67, 111, 110, 116, 101, 110, 116, 105, 111, 110, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 32, 116, 111, 32, 97, 99, 113, 117, 105, 114, 101, 32, 108, 111, 99, 107, 58, 32, 116, 105, 109, 101, 100, 32, 111, 117, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 57, 51, 57, 57, 50, 56, 49, 45, 48, 48, 97, 50, 45, 52, 98, 51, 57, 45, 98, 98, 51, 100, 45, 98, 50, 54, 52, 53, 98, 98, 98, 100, 98, 57, 51, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n Conte...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038436115614011392","occurred_at":"2026-03-30T02:00:16Z","signature_id":"481c4f5b5c4b9602a3adf1de8af50eb014ef0d1b21513a48a1df5be1e1d57903","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038436115614011392","occurred_at":"2026-03-30T02:00:16Z","signature_id":"592717ee6fed046c085b3e31f946cd65b6c891dfa12ffbcbaf7a9a65cfecaca9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038436115614011392","occurred_at":"2026-03-30T02:00:16Z","signature_id":"81a2e7894d0a2d52c2fdba0a657e1d3c5f15808605560f628ee0ed550bc32910","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038436115614011392","occurred_at":"2026-03-30T02:00:16Z","signature_id":"8a563dbb5aa2c07f1dd5f4f8dbe4169727c979c71a1a480f519a8c623e29f518","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038436115614011392","occurred_at":"2026-03-30T02:00:16Z","signature_id":"ab1af64cd2fd0073bcb42ad4fcc3577c446d580b5876bb1adc8b20453ca9a690","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"953e20e9d1db6d0f9c7e6c5ea18e02689f21a0690a145ecf8a28e443cb3a9c26","failure_pattern":"add password: error status code received from the API","search_query":"add password: error status code received from the API","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-24..2026-03-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:252]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e25420\u003e: \n add password: error status code received from the API\n {\n msg: \"add password: error status code received from the API\",\n err: \u003c*odataerrors.ODataError | 0xc00063e4e0\u003e{\n ApiError: {\n Message: \"\",\n ResponseStatusCode: 404,\n ResponseHeaders: {\n header: {\n headers: {\n \"x-ms-ags-diagnostic\": {\n \"{\\\"ServerInfo\\\":{\\\"DataCenter\\\":\\\"East US\\\",\\\"Slice\\\":\\\"E\\\",\\\"Ring\\\":\\\"5\\\",\\\"ScaleUnit\\\":\\\"007\\\",\\\"RoleInstance\\\":\\\"MN1PEPF00028332\\\"}}\": {},\n },\n \"x-ms-resource-unit\": {\"1\": {}},\n \"date\": {\n \"Tue, 31 Mar 2026 00:21:31 GMT\": {},\n },\n \"vary\": {\n \"Accept-Encoding\": {},\n },\n \"request-id\": {\n \"061ef776-ae17-49c6-bbac-b0d62d7d9704\": {},\n },\n \"cache-control\": {\"no-cache\": {}},\n \"content-type\": {\n \"application/json\": {},\n },\n \"client-request-id\": {\n \"061ef776-ae17-49c6-bbac-b0d62d7d9704\": {},\n },\n \"strict-transport-security\": {\n \"max-age=31536000\": {},\n },\n },\n },\n },\n },\n backingStore: \u003c*store.InMemoryBackingStore | 0xc000e253c0\u003e{\n returnOnlyChangedValues: false,\n initializationCompleted: false,\n store: {\n \"errorEscaped\": \u003c*odataerrors.MainError | 0xc000d26ce0\u003e{\n backingStore: \u003c*store.InMemoryBackingStore | 0xc000e253e0\u003e{\n returnOnlyChangedValues: false,\n initializationCompleted: false,\n store: {\n \"code\": \u003c*string | 0xc000d26b60\u003e\"Request_ResourceNotFound\",\n \"message\": \u003c*string | 0xc000d26bb0\u003e\"Resource '02691b9b-c5f7-45b0-9cff-7fc2c9805fcd' does not exist or one of its queried reference-property objects are not present.\",\n \"innerError\": \u003c*odataerrors.InnerError | 0xc000d26d40\u003e{\n backingStore: \u003c*store.InMemoryBackingStore | 0xc000e25400\u003e{\n returnOnlyChangedValues: false,\n initializationCompleted: false,\n store: {...: ..., ...: ...},\n subscribers: {},\n changedValues: {...: ..., ...: ...},\n },\n },\n },\n subscribers: {},\n changedValues: {\"code\": false, \"message\": false, \"innerError\": false},\n },\n },\n },\n subscribers: {},\n changedValues: {\n \"errorEscaped\": false,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:252]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e25420\u003e: \n add password: error status code received from the API\n {\n msg: \"add password: error status code received from the API\",\n err: \u003c*odataerrors.ODataError | 0xc00063e4e0\u003e{\n ApiError: {\n Message: \"\",\n ResponseStatusCode: 404,\n ResponseHeaders: {\n header: {\n headers: {\n \"x-ms-ags-diagnostic\": {\n \"{\\\"ServerInfo\\\":{\\\"DataCenter\\\":\\\"East US\\\",\\\"Slice\\\":\\\"E\\\",\\\"Ring\\\":\\\"5\\\",\\\"ScaleUnit\\\":\\\"007\\\",\\\"RoleInstance\\\":\\\"MN1PEPF00028332\\\"}}\": {},\n },\n \"x-ms-resource-unit\": {\"1\": {}},\n \"date\": {\n \"Tue, 31 Mar 2026 00:21:31 GMT\": {},\n },\n \"vary\": {\n \"Accept-Encoding\": {},\n },\n \"request-id\": {\n \"061ef776-ae17-49c6-bbac-b0d62d7d9704\": {},\n },\n \"cache-control\": {\"no-cache\": {}},\n \"content-type\": {\n \"application/json\": {},\n },\n \"client-request-id\": {\n \"061ef776-ae17-49c6-bbac-b0d62d7d9704\": {},\n },\n \"strict-transport-security\": {\n \"max-age=31536000\": {},\n },\n },\n },\n },\n },\n backingStore: \u003c*store.InMemoryBackingStore | 0xc000e253c0\u003e{\n returnOnlyChangedValues: false,\n initializationCompleted: false,\n store: {\n \"errorEscaped\": \u003c*odataerrors.MainError | 0xc000d26ce0\u003e{\n backingStore: \u003c*store.InMemoryBackingStore | 0xc000e253e0\u003e{\n returnOnlyChangedValues: false,\n initializationCompleted: false,\n store: {\n \"code\": \u003c*string | 0xc000d26b60\u003e\"Request_ResourceNotFound\",\n \"message\": \u003c*string | 0xc000d26bb0\u003e\"Resource '02691b9b-c5f7-45b0-9cff-7fc2c9805fcd' does not exist or one of its queried reference-property objects are not present.\",\n \"innerError\": \u003c*odataerrors.InnerError | 0xc000d26d40\u003e{\n backingStore: \u003c*store.InMemoryBackingStore | 0xc000e25400\u003e{\n returnOnlyChangedValues: false,\n initializationCompleted: false,\n store: {...: ..., ...: ...},\n subscribers: {},\n changedValues: {...: ..., ...: ...},\n },\n },\n },\n subscribers: {},\n changedValues: {\"code\": false, \"message\": false, \"innerError\": false},\n },\n },\n },\n subscribers: {},\n changedValues: {\n \"errorEscaped\": false,\n },\n },\n },\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038765373188214784","occurred_at":"2026-03-30T23:48:38Z","signature_id":"cc2bccb2a2b60ce37519536cdbed9cc7423921a24ea7ed594f472686dfb158ad","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}