{"meta":{"start_date":"2026-03-13","end_date":"2026-03-13","start_at":"2026-03-13T00:00:00Z","end_at":"2026-03-14T00:00:00Z","timezone":"UTC","generated_at":"2026-08-27T16:52:53Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":4,"failed_runs":3,"raw_occurrences":20,"matched_occurrences":20,"runs_affected":3},"rows":[{"environment":"stg","failure_pattern_id":"09d4ef692462a27c00db576d91310eaddda0a56e4545ba064bef74d9acf68c26","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message node pool \u003cnodepool\u003e '4.20.15' must not be greater than Control Plane version '4.20.8'; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":10,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":10,"after_last_push_count":20,"trend_counts":[0,0,0,0,0,10,10],"trend_range":"2026-03-07..2026-03-13 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:118]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00061c6e0\u003e: \n failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-s58clz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-s58clz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create NodePool np-1: failed starting nodepool creation \\\"np-1\\\" for cluster \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-s58clz\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-s58clz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00061c680\u003e{\n msg: \"failed starting nodepool creation \\\"np-1\\\" for cluster \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-s58clz\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-s58clz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000c48660\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Msedge-Ref\": [\n \"Ref A: 464A75CC61A1478A90E29E6DFB49BE37 Ref B: CH1AA2020610029 Ref C: 2026-03-13T01:49:31Z\",\n ],\n \"Content-Length\": [\"170\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"aed4189a-5711-491e-8bb6-acfacec34220\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"286\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260313T014932Z:5707dfab-8c93-438a-85f8-fe0b471a077a\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/307313a4-b300-4bb0-b030-114e8e644174\",\n ],\n \"Date\": [\n \"Fri, 13 Mar 2026 01:49:31 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"5707dfab-8c93-438a-85f8-fe0b471a077a\",\n ],\n \"Strict-Transpo...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:118]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00061c6e0\u003e: \n failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-s58clz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-s58clz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create NodePool np-1: failed starting nodepool creation \\\"np-1\\\" for cluster \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-s58clz\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-s58clz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00061c680\u003e{\n msg: \"failed starting nodepool creation \\\"np-1\\\" for cluster \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-s58clz\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-s58clz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000c48660\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Msedge-Ref\": [\n \"Ref A: 464A75CC61A1478A90E29E6DFB49BE37 Ref B: CH1AA2020610029 Ref C: 2026-03-13T01:49:31Z\",\n ],\n \"Content-Length\": [\"170\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"aed4189a-5711-491e-8bb6-acfacec34220\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"286\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260313T014932Z:5707dfab-8c93-438a-85f8-fe0b471a077a\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/307313a4-b300-4bb0-b030-114e8e644174\",\n ],\n \"Date\": [\n \"Fri, 13 Mar 2026 01:49:31 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"5707dfab-8c93-438a-85f8-fe0b471a077a\",\n ],\n \"Strict-Transpo...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:94]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000d94240\u003e: \n failed to create NodePool nodepool-128GiB: failed starting nodepool creation \"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-8cnnck\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-8cnnck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/nodePools/nodepool-128GiB\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create NodePool nodepool-128GiB: failed starting nodepool creation \\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-8cnnck\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-8cnnck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/nodePools/nodepool-128GiB\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000d94220\u003e{\n msg: \"failed starting nodepool creation \\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-8cnnck\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-8cnnck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/nodePools/nodepool-128GiB\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0011dc330\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: CE27CC750B4F4A1099A8FE01F39FD561 Ref B: CHI261140504029 Ref C: 2026-03-13T01:48:41Z\",\n ],\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"3a2f44cb-0316-432e-9f20-f948736553c2\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"e54bd54b-460d-43e4-a17a-c650ba3ec54f\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"3358\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/26c75988-1220-4415-af03-21c025e4c013\",\n ],\n \"Date\": [\n \"Fri, 13 Mar 2026 01:48:44 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Length\": [\"170\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2998\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260313T014844Z:e54bd54b-460d-43e4-a17a-c650ba3ec54f\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"P...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:94]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000d94240\u003e: \n failed to create NodePool nodepool-128GiB: failed starting nodepool creation \"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-8cnnck\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-8cnnck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/nodePools/nodepool-128GiB\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create NodePool nodepool-128GiB: failed starting nodepool creation \\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-8cnnck\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-8cnnck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/nodePools/nodepool-128GiB\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000d94220\u003e{\n msg: \"failed starting nodepool creation \\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-8cnnck\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-8cnnck/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/nodePools/nodepool-128GiB\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0011dc330\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: CE27CC750B4F4A1099A8FE01F39FD561 Ref B: CHI261140504029 Ref C: 2026-03-13T01:48:41Z\",\n ],\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"3a2f44cb-0316-432e-9f20-f948736553c2\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"e54bd54b-460d-43e4-a17a-c650ba3ec54f\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"3358\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/26c75988-1220-4415-af03-21c025e4c013\",\n ],\n \"Date\": [\n \"Fri, 13 Mar 2026 01:48:44 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Length\": [\"170\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2998\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260313T014844Z:e54bd54b-460d-43e4-a17a-c650ba3ec54f\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"P...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:102]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a03e00\u003e: \n failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-fph99j\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-fph99j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create NodePool np-1: failed starting nodepool creation \\\"np-1\\\" for cluster \\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fph99j\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-fph99j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a03de0\u003e{\n msg: \"failed starting nodepool creation \\\"np-1\\\" for cluster \\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fph99j\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-fph99j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0015b46f0\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Content-Length\": [\"170\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"X-Envoy-Upstream-Service-Time\": [\"2485\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260313T014759Z:b223bd31-5124-4336-8659-d621923b7581\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"985c3873-8557-481e-b7da-f41abf57b299\",\n ],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/223eff5f-1b5c-4425-89eb-063d6fc5f956\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"b223bd31-5124-4336-8659-d621923b7581\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: B9DD73C420794F7A8138A30F859A721A Ref B: CHI261140505042 Ref C: 2026-03-13T01:47:56Z\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:102]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a03e00\u003e: \n failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-fph99j\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-fph99j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create NodePool np-1: failed starting nodepool creation \\\"np-1\\\" for cluster \\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fph99j\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-fph99j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a03de0\u003e{\n msg: \"failed starting nodepool creation \\\"np-1\\\" for cluster \\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-fph99j\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-fph99j/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0015b46f0\u003e{\n ErrorCode: \"InvalidRequestContent\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Content-Length\": [\"170\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"X-Envoy-Upstream-Service-Time\": [\"2485\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260313T014759Z:b223bd31-5124-4336-8659-d621923b7581\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Ms-Error-Code\": [\n \"InvalidRequestContent\",\n ],\n \"X-Ms-Request-Id\": [\n \"985c3873-8557-481e-b7da-f41abf57b299\",\n ],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/223eff5f-1b5c-4425-89eb-063d6fc5f956\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"b223bd31-5124-4336-8659-d621923b7581\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: B9DD73C420794F7A8138A30F859A721A Ref B: CHI261140505042 Ref C: 2026-03-13T01:47:56Z\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032269313620054016","occurred_at":"2026-03-13T01:35:36Z","signature_id":"3429a8521dfff6089aad847db1449f1c8db70d5d9b8606794ed40df34a153bc4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032269313620054016","occurred_at":"2026-03-13T01:35:36Z","signature_id":"3d04710f418b161ee832e2231e0b0d5f900167301339584305861bf849eac93a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032269313620054016","occurred_at":"2026-03-13T01:35:36Z","signature_id":"6f8c4a6943ecb444ff6f8f3b5b68ca22f1d998b5fb6d688e579b2924b6b151ed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032269313620054016","occurred_at":"2026-03-13T01:35:36Z","signature_id":"89e14790ce2394e1705d671bc444728d34c85eab16fbf707275304ca8c3ad1d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032269313620054016","occurred_at":"2026-03-13T01:35:36Z","signature_id":"8ddd2d9cfd3e8fdc4e8077655561f4256cca9ebaa8b48ec3f0243af24a8386b3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032269313620054016","occurred_at":"2026-03-13T01:35:36Z","signature_id":"a0a75cf4edf92b208f8e35866af2ea4bf1acf4c74baf22636947f0ba24a6d4bd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032269313620054016","occurred_at":"2026-03-13T01:35:36Z","signature_id":"c8a8a15929af52a119de4a0160fc71edcb9060b2558d802620d4f4a7a8cd0a1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032269313620054016","occurred_at":"2026-03-13T01:35:36Z","signature_id":"e27809fe4f4048211934e6907279f518bf33670bc35efda50474356cf27c4f64","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032269313620054016","occurred_at":"2026-03-13T01:35:36Z","signature_id":"ed52f8d9eaa1e061eed0c76003e8f47e4210a53482586a10f06667f59ae31024","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032269313620054016","occurred_at":"2026-03-13T01:35:36Z","signature_id":"fe45756cf38c21e6331194047a0d687a71a22dfa10b0abc945252765077363f6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"e17b72ff49490f6e1940fa5deb5057f7012ebd4b2cee2f2d2e8b8d47e6784bb9","failure_pattern":"ERROR CODE: LockContention; detail message Failed to acquire lock: timed out; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LockContention","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":75,"anchor_occurrences":4,"after_last_push_count":6,"trend_counts":[2,0,0,0,0,0,4],"trend_range":"2026-03-07..2026-03-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-02"],"prior_runs_affected":4,"prior_last_seen_at":"2026-03-07T14:40:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000bea540\u003e: \n failed to create NodePool arm64-vm-np-1: failed starting nodepool creation \"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-d9rlj5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-d9rlj5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster/nodePools/arm64-vm-np-1\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: LockContention\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LockContention\",\n \"message\": \"Failed to acquire lock: timed out\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create NodePool arm64-vm-np-1: failed starting nodepool creation \\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-d9rlj5\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-d9rlj5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster/nodePools/arm64-vm-np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"LockContention\\\",\\n \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000bea520\u003e{\n msg: \"failed starting nodepool creation \\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-d9rlj5\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-d9rlj5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster/nodePools/arm64-vm-np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"LockContention\\\",\\n \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012ae630\u003e{\n ErrorCode: \"LockContention\",\n StatusCode: 503,\n RawResponse: {\n Status: \"503 Service Unavailable\",\n StatusCode: 503,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"Expires\": [\"-1\"],\n \"Retry-After\": [\"10\"],\n \"X-Envoy-Upstream-Service-Time\": [\"10019\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: ADF6D0781C9D48D8A5C81735B6310D6F Ref B: CHI261140512034 Ref C: 2026-03-13T07:15:12Z\",\n ],\n \"Date\": [\n \"Fri, 13 Mar 2026 07:15:21 GMT\",\n ],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/6b3f3604-a2f9-4117-8f92-506228f3e7af\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"186\"],\n \"X-Ms-Request-Id\": [\n \"8c39947a-1214-430d-8010-f0929133b44f\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"dafc4310-6617-45a5-b396-fc80a09e00a8\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000bea540\u003e: \n failed to create NodePool arm64-vm-np-1: failed starting nodepool creation \"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-d9rlj5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-d9rlj5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster/nodePools/arm64-vm-np-1\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: LockContention\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LockContention\",\n \"message\": \"Failed to acquire lock: timed out\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create NodePool arm64-vm-np-1: failed starting nodepool creation \\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-d9rlj5\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-d9rlj5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster/nodePools/arm64-vm-np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"LockContention\\\",\\n \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000bea520\u003e{\n msg: \"failed starting nodepool creation \\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-d9rlj5\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-d9rlj5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster/nodePools/arm64-vm-np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"LockContention\\\",\\n \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012ae630\u003e{\n ErrorCode: \"LockContention\",\n StatusCode: 503,\n RawResponse: {\n Status: \"503 Service Unavailable\",\n StatusCode: 503,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"Expires\": [\"-1\"],\n \"Retry-After\": [\"10\"],\n \"X-Envoy-Upstream-Service-Time\": [\"10019\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: ADF6D0781C9D48D8A5C81735B6310D6F Ref B: CHI261140512034 Ref C: 2026-03-13T07:15:12Z\",\n ],\n \"Date\": [\n \"Fri, 13 Mar 2026 07:15:21 GMT\",\n ],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/6b3f3604-a2f9-4117-8f92-506228f3e7af\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"186\"],\n \"X-Ms-Request-Id\": [\n \"8c39947a-1214-430d-8010-f0929133b44f\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"dafc4310-6617-45a5-b396-fc80a09e00a8\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n \u003c*errors.joinError | 0xc000fd40f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-26rkfj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: LockContention\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LockContention\",\n \"message\": \"Failed to acquire lock: timed out\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n errs: [\n \u003c*fmt.wrapError | 0xc001786300\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-26rkfj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"LockContention\\\",\\n \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0017862e0\u003e{\n msg: \"at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-26rkfj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"LockContention\\\",\\n \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0013d0540\u003e{\n ErrorCode: \"LockContention\",\n StatusCode: 503,\n RawResponse: {\n Status: \"503 Service Unavailable\",\n StatusCode: 503,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Ratelimit-Remaining-Subscription-Deletes\": [\"199\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: CC4F23FE193F4C1FB16C204ED434C9F4 Ref B: CHI261140510042 Ref C: 2026-03-13T03:16:52Z\",\n ],\n \"Expires\": [\"-1\"],\n \"Retry-After\": [\"10\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/fbb98992-521d-40e7-b146-c0dfb4a73360\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Deletes\": [\"2999\"],\n \"Date\": [\n \"Fri, 13 Mar 2026 03:17:01 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"LockContention\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"f2fd7682-45a9-47f9-a802-ee0323a08a2f\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260313T031702Z:f2fd7682-45a9-47f9-a802-ee0323a08a2f\",\n ],\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n \u003c*errors.joinError | 0xc000fd40f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-26rkfj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: LockContention\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LockContention\",\n \"message\": \"Failed to acquire lock: timed out\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n errs: [\n \u003c*fmt.wrapError | 0xc001786300\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-26rkfj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"LockContention\\\",\\n \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0017862e0\u003e{\n msg: \"at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-26rkfj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/node-limit-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"LockContention\\\",\\n \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0013d0540\u003e{\n ErrorCode: \"LockContention\",\n StatusCode: 503,\n RawResponse: {\n Status: \"503 Service Unavailable\",\n StatusCode: 503,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Ratelimit-Remaining-Subscription-Deletes\": [\"199\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: CC4F23FE193F4C1FB16C204ED434C9F4 Ref B: CHI261140510042 Ref C: 2026-03-13T03:16:52Z\",\n ],\n \"Expires\": [\"-1\"],\n \"Retry-After\": [\"10\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/fbb98992-521d-40e7-b146-c0dfb4a73360\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Deletes\": [\"2999\"],\n \"Date\": [\n \"Fri, 13 Mar 2026 03:17:01 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"LockContention\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"f2fd7682-45a9-47f9-a802-ee0323a08a2f\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260313T031702Z:f2fd7682-45a9-47f9-a802-ee0323a08a2f\",\n ],\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n \u003c*errors.joinError | 0xc000d300d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-v4pw7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: LockContention\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LockContention\",\n \"message\": \"Failed to acquire lock: timed out\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e024c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-v4pw7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"LockContention\\\",\\n \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e02480\u003e{\n msg: \"at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-v4pw7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"LockContention\\\",\\n \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000f90300\u003e{\n ErrorCode: \"LockContention\",\n StatusCode: 503,\n RawResponse: {\n Status: \"503 Service Unavailable\",\n StatusCode: 503,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Fri, 13 Mar 2026 03:19:28 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"Retry-After\": [\"10\"],\n \"X-Envoy-Upstream-Service-Time\": [\"10022\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Deletes\": [\"2999\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"8b4fb22e-bc20-46b1-8f98-d4392041161d\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: A9CAA84EC1294F488AB1EBE718FAA146 Ref B: CH1AA2020603009 Ref C: 2026-03-13T03:19:18Z\",\n ],\n \"Content-Length\": [\"186\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"51a5e370-92d6-4444-ab52-1b5412da1e41\",\n ],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/c7e2a0d7-ccf1-4903-af7f-e49949aa8449\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260313T031929Z:8b4fb22e-bc20-46b1-8f98-d4392041161d\",\n ],\n \"Strict-Transport-Security\": [\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n \u003c*errors.joinError | 0xc000d300d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-v4pw7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: LockContention\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LockContention\",\n \"message\": \"Failed to acquire lock: timed out\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n errs: [\n \u003c*fmt.wrapError | 0xc000e024c0\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-v4pw7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"LockContention\\\",\\n \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e02480\u003e{\n msg: \"at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-v4pw7s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"LockContention\\\",\\n \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000f90300\u003e{\n ErrorCode: \"LockContention\",\n StatusCode: 503,\n RawResponse: {\n Status: \"503 Service Unavailable\",\n StatusCode: 503,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Date\": [\n \"Fri, 13 Mar 2026 03:19:28 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"Retry-After\": [\"10\"],\n \"X-Envoy-Upstream-Service-Time\": [\"10022\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Deletes\": [\"2999\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"8b4fb22e-bc20-46b1-8f98-d4392041161d\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: A9CAA84EC1294F488AB1EBE718FAA146 Ref B: CH1AA2020603009 Ref C: 2026-03-13T03:19:18Z\",\n ],\n \"Content-Length\": [\"186\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"51a5e370-92d6-4444-ab52-1b5412da1e41\",\n ],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/c7e2a0d7-ccf1-4903-af7f-e49949aa8449\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260313T031929Z:8b4fb22e-bc20-46b1-8f98-d4392041161d\",\n ],\n \"Strict-Transport-Security\": [\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032350070564720640","occurred_at":"2026-03-13T06:56:30Z","signature_id":"78279ea9fdc4ea44e17d99ceac9c668de66fec3b55e1ced271994d0a2dd7f2d9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032291435486646272","occurred_at":"2026-03-13T03:03:30Z","signature_id":"41e8a5bcaf222468125139b5c143a7c121637c54c7a6359f5aee7f51fb75fb3f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032291435486646272","occurred_at":"2026-03-13T03:03:30Z","signature_id":"6f19acd2bc1ce60e3c9830bb18526bb3cd2f667731420489c51fef9edc7dc1da","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032269313620054016","occurred_at":"2026-03-13T01:35:36Z","signature_id":"c8162172865e93d45e84d14a8f86e6a9b8ad704475567e9a0b98ca8d25f0f1ca","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7f0e44476112ce4e675f687843407c03482d80d265239de5b2ddbd45c441a04e","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"customer-rg-4tpb7n\" to finish deleting: context canceled","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":75,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-03-07..2026-03-13 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0017ea2e8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-f85kxr\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d15e00\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-f85kxr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d15de0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-f85kxr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d15dc0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-f85kxr\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb08a0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0017ea2e8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-f85kxr\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000d15e00\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-f85kxr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d15de0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-f85kxr\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000d15dc0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-f85kxr\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb08a0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000da81c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-bbt9j2\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000b35120\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-bbt9j2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b35100\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-bbt9j2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b350e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-bbt9j2\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb08a0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc000da81c8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-bbt9j2\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000b35120\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-bbt9j2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b35100\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-bbt9j2\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc000b350e0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-bbt9j2\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb08a0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0015623f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listcwcr6n\" in resourcegroup=\"ea-list-kd7958\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001681600\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listcwcr6n\\\" in resourcegroup=\\\"ea-list-kd7958\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0016815e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listcwcr6n\\\" in resourcegroup=\\\"ea-list-kd7958\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0016815c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listcwcr6n\\\" in resourcegroup=\\\"ea-list-kd7958\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb08a0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n \u003c*errors.joinError | 0xc0015623f0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listcwcr6n\" in resourcegroup=\"ea-list-kd7958\" to finish deleting: context canceled\n {\n errs: [\n \u003c*fmt.wrapError | 0xc001681600\u003e{\n msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listcwcr6n\\\" in resourcegroup=\\\"ea-list-kd7958\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0016815e0\u003e{\n msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-listcwcr6n\\\" in resourcegroup=\\\"ea-list-kd7958\\\" to finish deleting: context canceled\",\n err: \u003c*fmt.wrapError | 0xc0016815c0\u003e{\n msg: \"failed waiting for hcpCluster=\\\"ea-listcwcr6n\\\" in resourcegroup=\\\"ea-list-kd7958\\\" to finish deleting: context canceled\",\n err: \u003c*errors.errorString | 0xb08a0c0\u003e{\n s: \"context canceled\",\n },\n },\n },\n },\n ],\n }\noccurred \nfail [:0]: A node timeout occurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032350070564720640","occurred_at":"2026-03-13T06:56:30Z","signature_id":"30ff88ef4207d2b37415bcafd1dd7fdbc0e83497c922de4901ce72eae510a2be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032350070564720640","occurred_at":"2026-03-13T06:56:30Z","signature_id":"75cf4003641c306d9bdaa35f94a7b2ca743352f27c81876cff476ebe5cb018df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032291435486646272","occurred_at":"2026-03-13T03:03:30Z","signature_id":"2322f06257d5def79bb5550003949c848328c6472c80b0471f5ddacdd717cb69","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032269313620054016","occurred_at":"2026-03-13T01:35:36Z","signature_id":"9117bf80f2d1d517acb2e9b07b614c3c4e18b17dfbc4cd130658f1b80b3f8ab7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"604745dc1eb894c4ac98add4f5ede2030b2008c5ab8972455c0f282bc06425ad","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade using the candidate channel","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":50,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,1,2],"trend_range":"2026-03-07..2026-03-13 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade using the candidate channel","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: Timed out after 2400.005s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001085770\u003e: \n VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \"4.20.15\"\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a12720\u003e{\n msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n err: \u003c*errors.errorString | 0xc0008a2660\u003e{\n s: \"no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n },\n },\n ],\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: Timed out after 2400.005s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001085770\u003e: \n VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \"4.20.15\"\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000a12720\u003e{\n msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n err: \u003c*errors.errorString | 0xc0008a2660\u003e{\n s: \"no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n },\n },\n ],\n }","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: Timed out after 2400.009s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0011582e8\u003e: \n VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \"4.20.15\"\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000ac67c0\u003e{\n msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n err: \u003c*errors.errorString | 0xc0001427d0\u003e{\n s: \"no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n },\n },\n ],\n }\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: Timed out after 2400.009s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0011582e8\u003e: \n VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \"4.20.15\"\n {\n errs: [\n \u003c*fmt.wrapError | 0xc000ac67c0\u003e{\n msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n err: \u003c*errors.errorString | 0xc0001427d0\u003e{\n s: \"no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n },\n },\n ],\n }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032291435486646272","occurred_at":"2026-03-13T03:03:30Z","signature_id":"54fc4f988e82abcbf5c5baa68953e10e2e85db7e10e5990106e714634b25f1ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2032269313620054016","occurred_at":"2026-03-13T01:35:36Z","signature_id":"7d85294b92b32536ed2186903512778127585caff2405ddc45754e35d52943eb","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}