{"meta":{"start_date":"2026-03-09","end_date":"2026-03-15","start_at":"2026-03-09T00:00:00Z","end_at":"2026-03-16T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T19:00:43Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":46,"failed_runs":31,"raw_occurrences":58,"matched_occurrences":56,"runs_affected":29},"rows":[{"environment":"int","failure_pattern_id":"2bdca75485ae62cfd6bf50594432970ef0c662e6c4c0217737e657a165a51784","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message node pool \u003cnodepool\u003e '4.20.15' must not be greater than Control Plane version '4.20.8'; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":9,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.1739130434782608,"anchor_occurrences":9,"after_last_push_count":9,"trend_counts":[0,0,0,0,9,0,0,0],"trend_range":"2026-03-08..2026-03-15 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:118]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00171c6c0\u003e: \n    failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-g975c2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-g975c2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create NodePool np-1: failed starting nodepool creation \\\"np-1\\\" for cluster \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-g975c2\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-g975c2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00171c6a0\u003e{\n            msg: \"failed starting nodepool creation \\\"np-1\\\" for cluster \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-g975c2\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-g975c2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0009663c0\u003e{\n                ErrorCode: \"InvalidRequestContent\",\n                StatusCode: 400,\n                RawResponse: {\n                    Status: \"400 Bad Request\",\n                    StatusCode: 400,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Length\": [\"170\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"cf3a6b30-b9f9-40f5-860b-db7a4db99729\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"UKSOUTH:20260312T231200Z:cf3a6b30-b9f9-40f5-860b-db7a4db99729\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 1334633E7ABC4C9D961015632B4669DE Ref B: CHI261140502060 Ref C: 2026-03-12T23:11:59Z\",\n                        ],\n                        \"Date\": [\n                            \"Thu, 12 Mar 2026 23:11:59 GMT\",\n                        ],\n                        \"X-Envoy-Upstream-Service-Time\": [\"201\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/67d0f8c8-def2-4e27-b339-fb62d5a3d936\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Error-Code\": [\n                            \"InvalidRequestContent\",\n                        ],\n                        \"X-Ms-Request-Id\": [\n                            \"f425ba51-7086-4447-8016-171717a6c5ab\",\n                    ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:118]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00171c6c0\u003e: \n    failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-g975c2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-g975c2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create NodePool np-1: failed starting nodepool creation \\\"np-1\\\" for cluster \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-g975c2\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-g975c2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00171c6a0\u003e{\n            msg: \"failed starting nodepool creation \\\"np-1\\\" for cluster \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-g975c2\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-g975c2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0009663c0\u003e{\n                ErrorCode: \"InvalidRequestContent\",\n                StatusCode: 400,\n                RawResponse: {\n                    Status: \"400 Bad Request\",\n                    StatusCode: 400,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Length\": [\"170\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"cf3a6b30-b9f9-40f5-860b-db7a4db99729\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"UKSOUTH:20260312T231200Z:cf3a6b30-b9f9-40f5-860b-db7a4db99729\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 1334633E7ABC4C9D961015632B4669DE Ref B: CHI261140502060 Ref C: 2026-03-12T23:11:59Z\",\n                        ],\n                        \"Date\": [\n                            \"Thu, 12 Mar 2026 23:11:59 GMT\",\n                        ],\n                        \"X-Envoy-Upstream-Service-Time\": [\"201\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/67d0f8c8-def2-4e27-b339-fb62d5a3d936\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Error-Code\": [\n                            \"InvalidRequestContent\",\n                        ],\n                        \"X-Ms-Request-Id\": [\n                            \"f425ba51-7086-4447-8016-171717a6c5ab\",\n                    ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:114]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001408120\u003e: \n    failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-8t5md5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-8t5md5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create NodePool np-1: failed starting nodepool creation \\\"np-1\\\" for cluster \\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-8t5md5\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-8t5md5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc001408100\u003e{\n            msg: \"failed starting nodepool creation \\\"np-1\\\" for cluster \\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-8t5md5\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-8t5md5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000c90fc0\u003e{\n                ErrorCode: \"InvalidRequestContent\",\n                StatusCode: 400,\n                RawResponse: {\n                    Status: \"400 Bad Request\",\n                    StatusCode: 400,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Envoy-Upstream-Service-Time\": [\"2649\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"UKSOUTH:20260312T231012Z:014d9526-d846-4eb1-91c9-ca8a5c50bfbd\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 8080CB14631643F1BD10EDD83999FADD Ref B: CHI261140513054 Ref C: 2026-03-12T23:10:10Z\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Length\": [\"170\"],\n                        \"Expires\": [\"-1\"],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/9e990a7a-bc1c-4fa5-84e6-6d350f3c8cfe\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Date\": [\n                            \"Thu, 12 Mar 2026 23:10:12 GMT\",\n                        ],\n                        \"X-Ms-Error-Code\": [\n                            \"InvalidRequestContent\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"014d9526-d846-4eb1-91c9-ca8a5c50bfbd\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Ty...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:114]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001408120\u003e: \n    failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-8t5md5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-8t5md5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create NodePool np-1: failed starting nodepool creation \\\"np-1\\\" for cluster \\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-8t5md5\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-8t5md5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc001408100\u003e{\n            msg: \"failed starting nodepool creation \\\"np-1\\\" for cluster \\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-8t5md5\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-8t5md5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/tls-endpoint-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000c90fc0\u003e{\n                ErrorCode: \"InvalidRequestContent\",\n                StatusCode: 400,\n                RawResponse: {\n                    Status: \"400 Bad Request\",\n                    StatusCode: 400,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Envoy-Upstream-Service-Time\": [\"2649\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"UKSOUTH:20260312T231012Z:014d9526-d846-4eb1-91c9-ca8a5c50bfbd\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 8080CB14631643F1BD10EDD83999FADD Ref B: CHI261140513054 Ref C: 2026-03-12T23:10:10Z\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Length\": [\"170\"],\n                        \"Expires\": [\"-1\"],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/9e990a7a-bc1c-4fa5-84e6-6d350f3c8cfe\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Date\": [\n                            \"Thu, 12 Mar 2026 23:10:12 GMT\",\n                        ],\n                        \"X-Ms-Error-Code\": [\n                            \"InvalidRequestContent\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"014d9526-d846-4eb1-91c9-ca8a5c50bfbd\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Ty...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:102]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00087e580\u003e: \n    failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-mq56rt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-mq56rt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create NodePool np-1: failed starting nodepool creation \\\"np-1\\\" for cluster \\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-mq56rt\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-mq56rt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00087e560\u003e{\n            msg: \"failed starting nodepool creation \\\"np-1\\\" for cluster \\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-mq56rt\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-mq56rt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000e0e090\u003e{\n                ErrorCode: \"InvalidRequestContent\",\n                StatusCode: 400,\n                RawResponse: {\n                    Status: \"400 Bad Request\",\n                    StatusCode: 400,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Request-Id\": [\n                            \"95277bb3-59de-4af9-a6be-d794c22ffea6\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: DECDB7E93FD643FFB79BD2D9D72D81DF Ref B: CHI261140510025 Ref C: 2026-03-12T22:52:58Z\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Length\": [\"170\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"UKSOUTH:20260312T225305Z:9b9678c6-264f-423d-bafc-7733fb8aa855\",\n                        ],\n                        \"X-Ms-Error-Code\": [\n                            \"InvalidRequestContent\",\n                        ],\n                        \"X-Envoy-Upstream-Service-Time\": [\"7627\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/a50444fe-08f0-4f32-b20a-20c19b8c9e0e\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"9b9678c6-264f-423d-bafc-7733fb8aa855\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n  ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:102]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00087e580\u003e: \n    failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-mq56rt\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-mq56rt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create NodePool np-1: failed starting nodepool creation \\\"np-1\\\" for cluster \\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-mq56rt\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-mq56rt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00087e560\u003e{\n            msg: \"failed starting nodepool creation \\\"np-1\\\" for cluster \\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-mq56rt\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-mq56rt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidRequestContent\\\",\\n    \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000e0e090\u003e{\n                ErrorCode: \"InvalidRequestContent\",\n                StatusCode: 400,\n                RawResponse: {\n                    Status: \"400 Bad Request\",\n                    StatusCode: 400,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Request-Id\": [\n                            \"95277bb3-59de-4af9-a6be-d794c22ffea6\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: DECDB7E93FD643FFB79BD2D9D72D81DF Ref B: CHI261140510025 Ref C: 2026-03-12T22:52:58Z\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Length\": [\"170\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"UKSOUTH:20260312T225305Z:9b9678c6-264f-423d-bafc-7733fb8aa855\",\n                        ],\n                        \"X-Ms-Error-Code\": [\n                            \"InvalidRequestContent\",\n                        ],\n                        \"X-Envoy-Upstream-Service-Time\": [\"7627\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/a50444fe-08f0-4f32-b20a-20c19b8c9e0e\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"9b9678c6-264f-423d-bafc-7733fb8aa855\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n  ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032218705995239424","occurred_at":"2026-03-12T22:14:30Z","signature_id":"0d302bd1eaa929a3233de857cb877068d6fda19624fab06e55718ea799ca62d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032218705995239424","occurred_at":"2026-03-12T22:14:30Z","signature_id":"2f3da2b73cf69b0e076e7f409aeaca1b308ac86418b5ab65c27dbff02227428f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032218705995239424","occurred_at":"2026-03-12T22:14:30Z","signature_id":"69aa6ad38be8c1b97122ecfa30194d7cd3644e3ab5ce63221f5ebe958f86f582","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032218705995239424","occurred_at":"2026-03-12T22:14:30Z","signature_id":"7e5db558d78fb45c605660dce96ef90ed71dd29ca36ee99c2d54084c355d094b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032218705995239424","occurred_at":"2026-03-12T22:14:30Z","signature_id":"84c6cbf80fb57bf41908643a4f1814d6c2dfdf207f36540040d78dd04f2bd297","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032218705995239424","occurred_at":"2026-03-12T22:14:30Z","signature_id":"854bd2a1f65b823098d7b717e41478b9b5e1dafb47cbf34945576ab916a43ee6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032218705995239424","occurred_at":"2026-03-12T22:14:30Z","signature_id":"bec18918cd70c49f7ad9ca6c2563de99c7bd15d99da5331673eccff5f320c7fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032218705995239424","occurred_at":"2026-03-12T22:14:30Z","signature_id":"d4b041e0e81d83a806095ad51d665752e1a469fd608baf7a9563305c1512da37","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032218705995239424","occurred_at":"2026-03-12T22:14:30Z","signature_id":"e5e43aa8b036468c250b998da4c93857baade7db0b446f29520280e44761e1e9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"52d9258f03a60774b49eaa70a514574e25b40d0eeb6ddfa13f252e24164f6be2","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-wfxt59\" to finish deleting: context canceled","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":6,"runs_affected":5,"failed_runs":5,"run_impact_percent":10.869565217391305,"anchor_occurrences":6,"after_last_push_count":6,"trend_counts":[0,0,0,1,2,3,0,0],"trend_range":"2026-03-08..2026-03-15 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes vm type NC4asT4v3 in a single cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001620390\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-vl49pp\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc00080bde0\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-vl49pp\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc00080bdc0\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-vl49pp\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc00080bda0\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-vl49pp\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb08a0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001620390\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-vl49pp\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc00080bde0\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-vl49pp\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc00080bdc0\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-vl49pp\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc00080bda0\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-vl49pp\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb08a0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000d462a0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-kvlw4b\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0008ff140\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-kvlw4b\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc0008ff120\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-kvlw4b\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc0008ff100\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-kvlw4b\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb08a0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000d462a0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-kvlw4b\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0008ff140\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-kvlw4b\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc0008ff120\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-kvlw4b\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc0008ff100\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-kvlw4b\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb08a0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000a38c90\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"list-test-cluster-2fj9s5\" in resourcegroup=\"cluster-listing-hdwkfd\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0007639e0\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"list-test-cluster-2fj9s5\\\" in resourcegroup=\\\"cluster-listing-hdwkfd\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc0007636c0\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"list-test-cluster-2fj9s5\\\" in resourcegroup=\\\"cluster-listing-hdwkfd\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000763560\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"list-test-cluster-2fj9s5\\\" in resourcegroup=\\\"cluster-listing-hdwkfd\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb08a0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000a38c90\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"list-test-cluster-2fj9s5\" in resourcegroup=\"cluster-listing-hdwkfd\" to finish deleting: context canceled\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0007639e0\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"list-test-cluster-2fj9s5\\\" in resourcegroup=\\\"cluster-listing-hdwkfd\\\" to finish deleting: context canceled\",\n                err: \u003c*fmt.wrapError | 0xc0007636c0\u003e{\n                    msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"list-test-cluster-2fj9s5\\\" in resourcegroup=\\\"cluster-listing-hdwkfd\\\" to finish deleting: context canceled\",\n                    err: \u003c*fmt.wrapError | 0xc000763560\u003e{\n                        msg: \"failed waiting for hcpCluster=\\\"list-test-cluster-2fj9s5\\\" in resourcegroup=\\\"cluster-listing-hdwkfd\\\" to finish deleting: context canceled\",\n                        err: \u003c*errors.errorString | 0xb08a0c0\u003e{\n                            s: \"context canceled\",\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred  \nfail [:0]: A node timeout occurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032330637133746176","occurred_at":"2026-03-13T05:39:17Z","signature_id":"c653b6c12aec5da60c1d11e3f7b77290288e2ee806aa4da91f4b927849edf2e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032330637133746176","occurred_at":"2026-03-13T05:39:17Z","signature_id":"f3a2caf1007cbcff1bec12add09b8099feb4b3f136b2b1fd09999bbae616646c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032295845155573760","occurred_at":"2026-03-13T03:21:02Z","signature_id":"108017e0217f09945a474164214d02830de0997389125699c3e979212026ff76","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032203554982203392","occurred_at":"2026-03-12T21:14:18Z","signature_id":"7f7d936a3963633186d29d2ada5fc63937e2290d83a6c29af149ef22bee2cea6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032126189862129664","occurred_at":"2026-03-12T16:06:53Z","signature_id":"df395833ef0baa4962459163c76ae67c6a8905c2a9404b62d65faaeb4fe81af1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2031742424082026496","occurred_at":"2026-03-11T14:41:44Z","signature_id":"20317d971b4b06150a38cca26ed194e611aa2dc2efea7c1c10a30d450f906984","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"8d06cad82b6f76b897e08d4e2537f6c6d4cc616ff64635c4d37783734841ca74","failure_pattern":"RevokeCredentialsAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-7z8gkd\" in resourcegroup=\"admin-credential-lifecycle-test-mf5bt8\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-7z8gkd in resource group admin-credential-lifecycle-test-mf5bt8, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":10.869565217391305,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,0,2,3],"trend_range":"2026-03-08..2026-03-15 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:193]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010d0990\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-pq72j7\" in resourcegroup=\"admin-credential-lifecycle-test-h2qqpf\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-pq72j7 in resource group admin-credential-lifecycle-test-h2qqpf, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-pq72j7\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-h2qqpf\\\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-pq72j7 in resource group admin-credential-lifecycle-test-h2qqpf, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000ba21f0\u003e{\n                s: \"timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-pq72j7 in resource group admin-credential-lifecycle-test-h2qqpf\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:193]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010d0990\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-pq72j7\" in resourcegroup=\"admin-credential-lifecycle-test-h2qqpf\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-pq72j7 in resource group admin-credential-lifecycle-test-h2qqpf, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-pq72j7\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-h2qqpf\\\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-pq72j7 in resource group admin-credential-lifecycle-test-h2qqpf, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000ba21f0\u003e{\n                s: \"timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-pq72j7 in resource group admin-credential-lifecycle-test-h2qqpf\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:193]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015288d0\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-xxmzq6\" in resourcegroup=\"admin-credential-lifecycle-test-gb2rdr\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-xxmzq6 in resource group admin-credential-lifecycle-test-gb2rdr, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-xxmzq6\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-gb2rdr\\\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-xxmzq6 in resource group admin-credential-lifecycle-test-gb2rdr, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000037e40\u003e{\n                s: \"timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-xxmzq6 in resource group admin-credential-lifecycle-test-gb2rdr\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:193]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015288d0\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-xxmzq6\" in resourcegroup=\"admin-credential-lifecycle-test-gb2rdr\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-xxmzq6 in resource group admin-credential-lifecycle-test-gb2rdr, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-xxmzq6\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-gb2rdr\\\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-xxmzq6 in resource group admin-credential-lifecycle-test-gb2rdr, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000037e40\u003e{\n                s: \"timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-xxmzq6 in resource group admin-credential-lifecycle-test-gb2rdr\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:193]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001148150\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-7z8gkd\" in resourcegroup=\"admin-credential-lifecycle-test-mf5bt8\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-7z8gkd in resource group admin-credential-lifecycle-test-mf5bt8, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-7z8gkd\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-mf5bt8\\\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-7z8gkd in resource group admin-credential-lifecycle-test-mf5bt8, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00050faa0\u003e{\n                s: \"timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-7z8gkd in resource group admin-credential-lifecycle-test-mf5bt8\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:193]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001148150\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-7z8gkd\" in resourcegroup=\"admin-credential-lifecycle-test-mf5bt8\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-7z8gkd in resource group admin-credential-lifecycle-test-mf5bt8, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-7z8gkd\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-mf5bt8\\\" to finish revoking creds, caused by: timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-7z8gkd in resource group admin-credential-lifecycle-test-mf5bt8, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00050faa0\u003e{\n                s: \"timeout '15.000000' minutes exceeded during RevokeCredentialsAndWait for cluster admin-cred-lifecycle-7z8gkd in resource group admin-credential-lifecycle-test-mf5bt8\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2033299216234713088","occurred_at":"2026-03-15T21:48:04Z","signature_id":"00607cd1cdc2bfbf4c88f9d3cb9c8afbddbede8d8e71cafe0aa2be5a2363e8ce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2033270683638697984","occurred_at":"2026-03-15T19:54:41Z","signature_id":"fec0608bb0cf46176c9fd684b0641db1827be0374b8c9a6c33982b7d92d5268c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2033020998986502144","occurred_at":"2026-03-15T03:22:32Z","signature_id":"441f5fc2141773a03a51020eead7cd46cf70942b43f8cb583b8948476820cc77","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032842010720735232","occurred_at":"2026-03-14T15:31:18Z","signature_id":"971bff6eab10d7bbdfe66218da1fb2f68e2cb9ff1fcf9a098d59cd86f941c022","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032804286143926272","occurred_at":"2026-03-14T13:01:24Z","signature_id":"59f620953c130864f7871412b35f32f392d1be29509a3ce0429270c8d33d8aa7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"29001e2ab3400d2423e59cccf5d3bc75b0bb1948c7c11c55b2b4e3162fb2cbea","failure_pattern":"not to be nil","search_query":"not to be nil","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":5,"runs_affected":3,"failed_runs":3,"run_impact_percent":6.521739130434782,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,0,0,5,0,0,0,0],"trend_range":"2026-03-08..2026-03-15 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:91]: Expected\n    \u003c*string | 0x0\u003e: nil\nnot to be nil\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:91]: Expected\n    \u003c*string | 0x0\u003e: nil\nnot to be nil","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:145]: Expected\n    \u003c*string | 0x0\u003e: nil\nnot to be nil\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:145]: Expected\n    \u003c*string | 0x0\u003e: nil\nnot to be nil"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2031775377503817728","occurred_at":"2026-03-11T16:52:53Z","signature_id":"fba1a6daaaa11df84b87d7f9ebfc1889d5025dd4fa57c6018f3a3dc38df0b823","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2031775377503817728","occurred_at":"2026-03-11T16:52:53Z","signature_id":"fba1a6daaaa11df84b87d7f9ebfc1889d5025dd4fa57c6018f3a3dc38df0b823","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2031675611008733184","occurred_at":"2026-03-11T10:16:26Z","signature_id":"fba1a6daaaa11df84b87d7f9ebfc1889d5025dd4fa57c6018f3a3dc38df0b823","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2031675611008733184","occurred_at":"2026-03-11T10:16:26Z","signature_id":"fba1a6daaaa11df84b87d7f9ebfc1889d5025dd4fa57c6018f3a3dc38df0b823","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2031571552545280000","occurred_at":"2026-03-11T03:22:57Z","signature_id":"fba1a6daaaa11df84b87d7f9ebfc1889d5025dd4fa57c6018f3a3dc38df0b823","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"234737fe1de0ab0d4753d9ece76af3c50d895013bb803ce30f31eba3838f12b6","failure_pattern":"ERROR CODE: InvalidIdentityValues; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidIdentityValues","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":8.695652173913043,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,4,0,0,0,0],"trend_range":"2026-03-08..2026-03-15 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:415]: Unexpected error:\n    \u003c*exported.ResponseError | 0xc000f97890\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-zpzk8c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidIdentityValues\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidIdentityValues\",\n        \"message\": \"Invalid value for the identities '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service'. The 'UserAssignedIdentities' property keys should only be empty null during PATCH requests.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        ErrorCode: \"InvalidIdentityValues\",\n        StatusCode: 400,\n        RawResponse: {\n            Status: \"400 Bad Request\",\n            StatusCode: 400,\n            Proto: \"HTTP/2.0\",\n            ProtoMajor: 2,\n            ProtoMinor: 0,\n            Header: {\n                \"X-Ms-Routing-Request-Id\": [\n                    \"UKSOUTH:20260311T151231Z:9c11778c-ff60-487b-a423-651593da8efe\",\n                ],\n                \"Strict-Transport-Security\": [\n                    \"max-age=31536000; includeSubDomains\",\n                ],\n                \"X-Msedge-Ref\": [\n                    \"Ref A: 4505E2BF16A0421FA071A149B08EF585 Ref B: CHI261140504060 Ref C: 2026-03-11T15:12:31Z\",\n                ],\n                \"Cache-Control\": [\"no-cache\"],\n                \"Content-Type\": [\n                    \"application/json; charset=utf-8\",\n                ],\n                \"X-Ms-Failure-Cause\": [\"gateway\"],\n                \"X-Content-Type-Options\": [\"nosniff\"],\n                \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                \"Date\": [\n                    \"Wed, 11 Mar 2026 15:12:31 GMT\",\n                ],\n                \"Pragma\": [\"no-cache\"],\n                \"Content-Length\": [\"1927\"],\n                \"Expires\": [\"-1\"],\n                \"X-Ms-Request-Id\": [\n                    \"9c11778c-ff60-487b-a423-651593da8efe\",\n                ],\n                \"X-Ms-Correlation-Request-Id\": [\n                    \"9c11778c-ff60-487b-a423-651593da8efe\",\n                ],\n            },\n            Body: \u003c*exported.nopClosingBytesReader | 0xc001100fc0\u003e{\n                s: \"{\\\"error\\\":{\\\"code\\\":\\\"InvalidIdentityValues\\\",\\\"message\\\":\\\"Invalid value for the identities '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service'. The 'UserAssignedIdentities' property keys should only be empty null during PATCH requests.\\\"}}\",\n                i: 0,\n            },\n            ContentLength: 1927,\n            TransferEncoding: nil,\n            Close: false,\n            Uncompressed: false,\n            Trailer: nil,\n            Request: {\n                Method: \"PUT\",\n                URL: {\n                    Scheme: \"https\",\n                    Opaque: \"\",\n                    User: nil,\n                    Host: \"management.azure.com\",\n                    Path: \"/subscriptions/64f0619f-ebc2-4156-...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:415]: Unexpected error:\n    \u003c*exported.ResponseError | 0xc000f97890\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-zpzk8c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidIdentityValues\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidIdentityValues\",\n        \"message\": \"Invalid value for the identities '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service'. The 'UserAssignedIdentities' property keys should only be empty null during PATCH requests.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        ErrorCode: \"InvalidIdentityValues\",\n        StatusCode: 400,\n        RawResponse: {\n            Status: \"400 Bad Request\",\n            StatusCode: 400,\n            Proto: \"HTTP/2.0\",\n            ProtoMajor: 2,\n            ProtoMinor: 0,\n            Header: {\n                \"X-Ms-Routing-Request-Id\": [\n                    \"UKSOUTH:20260311T151231Z:9c11778c-ff60-487b-a423-651593da8efe\",\n                ],\n                \"Strict-Transport-Security\": [\n                    \"max-age=31536000; includeSubDomains\",\n                ],\n                \"X-Msedge-Ref\": [\n                    \"Ref A: 4505E2BF16A0421FA071A149B08EF585 Ref B: CHI261140504060 Ref C: 2026-03-11T15:12:31Z\",\n                ],\n                \"Cache-Control\": [\"no-cache\"],\n                \"Content-Type\": [\n                    \"application/json; charset=utf-8\",\n                ],\n                \"X-Ms-Failure-Cause\": [\"gateway\"],\n                \"X-Content-Type-Options\": [\"nosniff\"],\n                \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                \"Date\": [\n                    \"Wed, 11 Mar 2026 15:12:31 GMT\",\n                ],\n                \"Pragma\": [\"no-cache\"],\n                \"Content-Length\": [\"1927\"],\n                \"Expires\": [\"-1\"],\n                \"X-Ms-Request-Id\": [\n                    \"9c11778c-ff60-487b-a423-651593da8efe\",\n                ],\n                \"X-Ms-Correlation-Request-Id\": [\n                    \"9c11778c-ff60-487b-a423-651593da8efe\",\n                ],\n            },\n            Body: \u003c*exported.nopClosingBytesReader | 0xc001100fc0\u003e{\n                s: \"{\\\"error\\\":{\\\"code\\\":\\\"InvalidIdentityValues\\\",\\\"message\\\":\\\"Invalid value for the identities '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-8/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service'. The 'UserAssignedIdentities' property keys should only be empty null during PATCH requests.\\\"}}\",\n                i: 0,\n            },\n            ContentLength: 1927,\n            TransferEncoding: nil,\n            Close: false,\n            Uncompressed: false,\n            Trailer: nil,\n            Request: {\n                Method: \"PUT\",\n                URL: {\n                    Scheme: \"https\",\n                    Opaque: \"\",\n                    User: nil,\n                    Host: \"management.azure.com\",\n                    Path: \"/subscriptions/64f0619f-ebc2-4156-...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:415]: Unexpected error:\n    \u003c*exported.ResponseError | 0xc0009bd380\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-nld688/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidIdentityValues\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidIdentityValues\",\n        \"message\": \"Invalid value for the identities '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service'. The 'UserAssignedIdentities' property keys should only be empty null during PATCH requests.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        ErrorCode: \"InvalidIdentityValues\",\n        StatusCode: 400,\n        RawResponse: {\n            Status: \"400 Bad Request\",\n            StatusCode: 400,\n            Proto: \"HTTP/2.0\",\n            ProtoMajor: 2,\n            ProtoMinor: 0,\n            Header: {\n                \"Cache-Control\": [\"no-cache\"],\n                \"Pragma\": [\"no-cache\"],\n                \"Content-Type\": [\n                    \"application/json; charset=utf-8\",\n                ],\n                \"Expires\": [\"-1\"],\n                \"X-Ms-Routing-Request-Id\": [\n                    \"UKSOUTH:20260311T130855Z:1f5ad0e5-72af-4f34-87a3-cbd696db9711\",\n                ],\n                \"Strict-Transport-Security\": [\n                    \"max-age=31536000; includeSubDomains\",\n                ],\n                \"X-Content-Type-Options\": [\"nosniff\"],\n                \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                \"Content-Length\": [\"1937\"],\n                \"X-Ms-Failure-Cause\": [\"gateway\"],\n                \"X-Ms-Request-Id\": [\n                    \"1f5ad0e5-72af-4f34-87a3-cbd696db9711\",\n                ],\n                \"X-Ms-Correlation-Request-Id\": [\n                    \"1f5ad0e5-72af-4f34-87a3-cbd696db9711\",\n                ],\n                \"X-Msedge-Ref\": [\n                    \"Ref A: 79888FDC7E984BA8A98CF39D24F6722A Ref B: CH1AA2020610011 Ref C: 2026-03-11T13:08:55Z\",\n                ],\n                \"Date\": [\n                    \"Wed, 11 Mar 2026 13:08:54 GMT\",\n                ],\n            },\n            Body: \u003c*exported.nopClosingBytesReader | 0xc000d91100\u003e{\n                s: \"{\\\"error\\\":{\\\"code\\\":\\\"InvalidIdentityValues\\\",\\\"message\\\":\\\"Invalid value for the identities '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service'. The 'UserAssignedIdentities' property keys should only be empty null during PATCH requests.\\\"}}\",\n                i: 0,\n            },\n            ContentLength: 1937,\n            TransferEncoding: nil,\n            Close: false,\n            Uncompressed: false,\n            Trailer: nil,\n            Request: {\n                Method: \"PUT\",\n                URL: {\n                    Scheme: \"https\",\n                    Opaque: \"\",\n                    User: nil,\n                    Host: \"management.azure.com\",\n                    Path: \"/subscriptions/64f0619f-...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:415]: Unexpected error:\n    \u003c*exported.ResponseError | 0xc0009bd380\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-nld688/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidIdentityValues\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidIdentityValues\",\n        \"message\": \"Invalid value for the identities '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service'. The 'UserAssignedIdentities' property keys should only be empty null during PATCH requests.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        ErrorCode: \"InvalidIdentityValues\",\n        StatusCode: 400,\n        RawResponse: {\n            Status: \"400 Bad Request\",\n            StatusCode: 400,\n            Proto: \"HTTP/2.0\",\n            ProtoMajor: 2,\n            ProtoMinor: 0,\n            Header: {\n                \"Cache-Control\": [\"no-cache\"],\n                \"Pragma\": [\"no-cache\"],\n                \"Content-Type\": [\n                    \"application/json; charset=utf-8\",\n                ],\n                \"Expires\": [\"-1\"],\n                \"X-Ms-Routing-Request-Id\": [\n                    \"UKSOUTH:20260311T130855Z:1f5ad0e5-72af-4f34-87a3-cbd696db9711\",\n                ],\n                \"Strict-Transport-Security\": [\n                    \"max-age=31536000; includeSubDomains\",\n                ],\n                \"X-Content-Type-Options\": [\"nosniff\"],\n                \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                \"Content-Length\": [\"1937\"],\n                \"X-Ms-Failure-Cause\": [\"gateway\"],\n                \"X-Ms-Request-Id\": [\n                    \"1f5ad0e5-72af-4f34-87a3-cbd696db9711\",\n                ],\n                \"X-Ms-Correlation-Request-Id\": [\n                    \"1f5ad0e5-72af-4f34-87a3-cbd696db9711\",\n                ],\n                \"X-Msedge-Ref\": [\n                    \"Ref A: 79888FDC7E984BA8A98CF39D24F6722A Ref B: CH1AA2020610011 Ref C: 2026-03-11T13:08:55Z\",\n                ],\n                \"Date\": [\n                    \"Wed, 11 Mar 2026 13:08:54 GMT\",\n                ],\n            },\n            Body: \u003c*exported.nopClosingBytesReader | 0xc000d91100\u003e{\n                s: \"{\\\"error\\\":{\\\"code\\\":\\\"InvalidIdentityValues\\\",\\\"message\\\":\\\"Invalid value for the identities '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-67/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service'. The 'UserAssignedIdentities' property keys should only be empty null during PATCH requests.\\\"}}\",\n                i: 0,\n            },\n            ContentLength: 1937,\n            TransferEncoding: nil,\n            Close: false,\n            Uncompressed: false,\n            Trailer: nil,\n            Request: {\n                Method: \"PUT\",\n                URL: {\n                    Scheme: \"https\",\n                    Opaque: \"\",\n                    User: nil,\n                    Host: \"management.azure.com\",\n                    Path: \"/subscriptions/64f0619f-...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:415]: Unexpected error:\n    \u003c*exported.ResponseError | 0xc00087cb10\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-q89nkn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidIdentityValues\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidIdentityValues\",\n        \"message\": \"Invalid value for the identities '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service'. The 'UserAssignedIdentities' property keys should only be empty null during PATCH requests.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        ErrorCode: \"InvalidIdentityValues\",\n        StatusCode: 400,\n        RawResponse: {\n            Status: \"400 Bad Request\",\n            StatusCode: 400,\n            Proto: \"HTTP/2.0\",\n            ProtoMajor: 2,\n            ProtoMinor: 0,\n            Header: {\n                \"Expires\": [\"-1\"],\n                \"X-Ms-Request-Id\": [\n                    \"5d0f1a83-e9e3-4322-9402-079527a36e43\",\n                ],\n                \"X-Ms-Correlation-Request-Id\": [\n                    \"5d0f1a83-e9e3-4322-9402-079527a36e43\",\n                ],\n                \"Strict-Transport-Security\": [\n                    \"max-age=31536000; includeSubDomains\",\n                ],\n                \"X-Msedge-Ref\": [\n                    \"Ref A: 7799C63EF4DB4BDE88CFAD7F106271DB Ref B: CHI261140512040 Ref C: 2026-03-11T06:03:42Z\",\n                ],\n                \"Pragma\": [\"no-cache\"],\n                \"Content-Length\": [\"1947\"],\n                \"X-Ms-Failure-Cause\": [\"gateway\"],\n                \"X-Ms-Routing-Request-Id\": [\n                    \"UKSOUTH:20260311T060342Z:5d0f1a83-e9e3-4322-9402-079527a36e43\",\n                ],\n                \"X-Content-Type-Options\": [\"nosniff\"],\n                \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                \"Date\": [\n                    \"Wed, 11 Mar 2026 06:03:41 GMT\",\n                ],\n                \"Cache-Control\": [\"no-cache\"],\n                \"Content-Type\": [\n                    \"application/json; charset=utf-8\",\n                ],\n            },\n            Body: \u003c*exported.nopClosingBytesReader | 0xc00011d160\u003e{\n                s: \"{\\\"error\\\":{\\\"code\\\":\\\"InvalidIdentityValues\\\",\\\"message\\\":\\\"Invalid value for the identities '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service'. The 'UserAssignedIdentities' property keys should only be empty null during PATCH requests.\\\"}}\",\n                i: 0,\n            },\n            ContentLength: 1947,\n            TransferEncoding: nil,\n            Close: false,\n            Uncompressed: false,\n            Trailer: nil,\n            Request: {\n                Method: \"PUT\",\n                URL: {\n                    Scheme: \"https\",\n                    Opaque: \"\",\n                    User: nil,\n                    Host: \"management.azure.com\",\n                    Path: \"/subscriptions...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:415]: Unexpected error:\n    \u003c*exported.ResponseError | 0xc00087cb10\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-q89nkn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidIdentityValues\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidIdentityValues\",\n        \"message\": \"Invalid value for the identities '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service'. The 'UserAssignedIdentities' property keys should only be empty null during PATCH requests.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        ErrorCode: \"InvalidIdentityValues\",\n        StatusCode: 400,\n        RawResponse: {\n            Status: \"400 Bad Request\",\n            StatusCode: 400,\n            Proto: \"HTTP/2.0\",\n            ProtoMajor: 2,\n            ProtoMinor: 0,\n            Header: {\n                \"Expires\": [\"-1\"],\n                \"X-Ms-Request-Id\": [\n                    \"5d0f1a83-e9e3-4322-9402-079527a36e43\",\n                ],\n                \"X-Ms-Correlation-Request-Id\": [\n                    \"5d0f1a83-e9e3-4322-9402-079527a36e43\",\n                ],\n                \"Strict-Transport-Security\": [\n                    \"max-age=31536000; includeSubDomains\",\n                ],\n                \"X-Msedge-Ref\": [\n                    \"Ref A: 7799C63EF4DB4BDE88CFAD7F106271DB Ref B: CHI261140512040 Ref C: 2026-03-11T06:03:42Z\",\n                ],\n                \"Pragma\": [\"no-cache\"],\n                \"Content-Length\": [\"1947\"],\n                \"X-Ms-Failure-Cause\": [\"gateway\"],\n                \"X-Ms-Routing-Request-Id\": [\n                    \"UKSOUTH:20260311T060342Z:5d0f1a83-e9e3-4322-9402-079527a36e43\",\n                ],\n                \"X-Content-Type-Options\": [\"nosniff\"],\n                \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                \"Date\": [\n                    \"Wed, 11 Mar 2026 06:03:41 GMT\",\n                ],\n                \"Cache-Control\": [\"no-cache\"],\n                \"Content-Type\": [\n                    \"application/json; charset=utf-8\",\n                ],\n            },\n            Body: \u003c*exported.nopClosingBytesReader | 0xc00011d160\u003e{\n                s: \"{\\\"error\\\":{\\\"code\\\":\\\"InvalidIdentityValues\\\",\\\"message\\\":\\\"Invalid value for the identities '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/disk-csi-driver,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/image-registry,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-int-143/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service'. The 'UserAssignedIdentities' property keys should only be empty null during PATCH requests.\\\"}}\",\n                i: 0,\n            },\n            ContentLength: 1947,\n            TransferEncoding: nil,\n            Close: false,\n            Uncompressed: false,\n            Trailer: nil,\n            Request: {\n                Method: \"PUT\",\n                URL: {\n                    Scheme: \"https\",\n                    Opaque: \"\",\n                    User: nil,\n                    Host: \"management.azure.com\",\n                    Path: \"/subscriptions...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2031742424082026496","occurred_at":"2026-03-11T14:41:44Z","signature_id":"8106c33fe5d18db952ae6b3ec4026f5a7c7f7b13b85b2e98174a2e4ede85f771","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2031711680643207168","occurred_at":"2026-03-11T12:39:46Z","signature_id":"ad29400a287a88f2d0d248e5fc14cf3c90cdb47287bf546e6b0ee1db34b142f0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2031604521683128320","occurred_at":"2026-03-11T05:33:57Z","signature_id":"7a08b280f6c0f60d10ed2e5627d3dedf6084a7ced2766fb96822ca0cf620882b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2031571552545280000","occurred_at":"2026-03-11T03:22:57Z","signature_id":"744f6b1e3d202963688b218a6448b3b28869ae19c41e84ed02ebe5391d23090f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"0ece011306dab74cd22dd19f1f3c89bbdb930a2321b61be59e4a6a90aae6e3eb","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-6wd77s, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-6wd77s\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-6wd77s, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":8.695652173913043,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,1,0,1,1,0,1,0],"trend_range":"2026-03-08..2026-03-15 UTC","prior_windows_present":2,"prior_window_starts":["2026-02-23","2026-03-02"],"prior_runs_affected":16,"prior_last_seen_at":"2026-03-08T23:23:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009aed80\u003e: \n    failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-ccshfw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-ccshfw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000f45af0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0009aed50\u003e{\n                msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-ccshfw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000f45af0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009aed80\u003e: \n    failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-ccshfw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-ccshfw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000f45af0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0009aed50\u003e{\n                msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-ccshfw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000f45af0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-ccshfw\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00032e8d0\u003e: \n    failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-zkjhtw, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-zkjhtw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-zkjhtw, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-zkjhtw, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-zkjhtw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-zkjhtw, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0014f4be0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-zkjhtw\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00032e8a0\u003e{\n                msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-zkjhtw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-zkjhtw, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0014f4be0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-zkjhtw\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00032e8d0\u003e: \n    failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-zkjhtw, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-zkjhtw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-zkjhtw, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-zkjhtw, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-zkjhtw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-zkjhtw, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0014f4be0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-zkjhtw\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00032e8a0\u003e{\n                msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-zkjhtw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-zkjhtw, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0014f4be0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-zkjhtw\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010ea060\u003e: \n    failed to create HCP cluster list-test-cluster-ccxwqz, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ccxwqz in resource group cluster-listing-d94s94, error: failed waiting for cluster=\"list-test-cluster-ccxwqz\" in resourcegroup=\"cluster-listing-d94s94\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ccxwqz in resource group cluster-listing-d94s94, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster list-test-cluster-ccxwqz, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ccxwqz in resource group cluster-listing-d94s94, error: failed waiting for cluster=\\\"list-test-cluster-ccxwqz\\\" in resourcegroup=\\\"cluster-listing-d94s94\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ccxwqz in resource group cluster-listing-d94s94, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000a83a70\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ccxwqz in resource group cluster-listing-d94s94\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0010ea030\u003e{\n                msg: \"failed waiting for cluster=\\\"list-test-cluster-ccxwqz\\\" in resourcegroup=\\\"cluster-listing-d94s94\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ccxwqz in resource group cluster-listing-d94s94, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000a83a70\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ccxwqz in resource group cluster-listing-d94s94\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010ea060\u003e: \n    failed to create HCP cluster list-test-cluster-ccxwqz, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ccxwqz in resource group cluster-listing-d94s94, error: failed waiting for cluster=\"list-test-cluster-ccxwqz\" in resourcegroup=\"cluster-listing-d94s94\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ccxwqz in resource group cluster-listing-d94s94, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster list-test-cluster-ccxwqz, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ccxwqz in resource group cluster-listing-d94s94, error: failed waiting for cluster=\\\"list-test-cluster-ccxwqz\\\" in resourcegroup=\\\"cluster-listing-d94s94\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ccxwqz in resource group cluster-listing-d94s94, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000a83a70\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ccxwqz in resource group cluster-listing-d94s94\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0010ea030\u003e{\n                msg: \"failed waiting for cluster=\\\"list-test-cluster-ccxwqz\\\" in resourcegroup=\\\"cluster-listing-d94s94\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ccxwqz in resource group cluster-listing-d94s94, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000a83a70\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-ccxwqz in resource group cluster-listing-d94s94\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032610884349595648","occurred_at":"2026-03-14T00:12:53Z","signature_id":"37269be883771bcdfa1b48c6a53e74a57dd276f0cb9c6f8b96f16a957b572578","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2031998313082916864","occurred_at":"2026-03-12T07:38:45Z","signature_id":"bb2f93a3c1b3e7f67c279551302f559a792592b42235dfa5992ba20c3097dd6e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2031521952853659648","occurred_at":"2026-03-11T00:05:52Z","signature_id":"07168bc9273e7f037f81a45921a6d3f9cc84eb3ef69c8dc67884232e7ce2c2de","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2030922443123593216","occurred_at":"2026-03-09T08:23:37Z","signature_id":"520b6a51c7d296789ca5b59353df1cf79bcad5ab6130fc5bba661df54c570b0c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"8be64044d6c01fd1b4bb536c29c4b3014256d4589d1fa91f7525668e9f395612","failure_pattern":"ERROR CODE: LockContention; detail message Failed to acquire lock: timed out; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LockContention","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":6.521739130434782,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,1,0,3,0,0,0],"trend_range":"2026-03-08..2026-03-15 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:118]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00038c420\u003e: \n    failed creating external auth \"ea-list\" in resourcegroup=\"ea-list-9lhb8x\" for cluster=\"ea-listgv59sm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-9lhb8x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listgv59sm/externalAuths/ea-list\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: LockContention\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LockContention\",\n        \"message\": \"Failed to acquire lock: timed out\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed creating external auth \\\"ea-list\\\" in resourcegroup=\\\"ea-list-9lhb8x\\\" for cluster=\\\"ea-listgv59sm\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-9lhb8x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listgv59sm/externalAuths/ea-list\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"LockContention\\\",\\n    \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc00076e300\u003e{\n            ErrorCode: \"LockContention\",\n            StatusCode: 503,\n            RawResponse: {\n                Status: \"503 Service Unavailable\",\n                StatusCode: 503,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Retry-After\": [\"10\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/4616b9c4-eae5-42c0-b42e-e60136e4b27a\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Date\": [\n                        \"Thu, 12 Mar 2026 22:53:06 GMT\",\n                    ],\n                    \"Content-Length\": [\"186\"],\n                    \"X-Ms-Request-Id\": [\n                        \"0daaf6b9-dea5-419a-a781-f5d3f0c88c48\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Error-Code\": [\"LockContention\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: DD91E82BD13A4C8F974F7F6335BA4D4F Ref B: CHI261140506025 Ref C: 2026-03-12T22:52:57Z\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"10007\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"132f0482-3ab7-4255-98f8-45a73cf0c24c\",\n                    ],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260312T225307Z:132f0482-3ab7-4255-98f8-45a73cf0c24c\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc00038c200\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 76, 111, 99, 107, 67, 111, 110, 116, 101, 110, 116, 105, 111, 110, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 32, 116, 111, 32, 97, 99, 113, 117, 105, 114, 101, 32, 108, 111, 99, 107, 58, 32, 116, 105, 109, 101, 100, 32, 111, 117, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 54, 52, 102, 48, 54, 49, 57, 102, 45, 101, 98, 99, 50, 45, 52, 49, 53, 54, 45, 57, 100, 57, 49, 45, 99, 52, 99, 55, 56, 49, 100, 101, 55, 101, 53, 52, 34, 10, 32, 32, 32, 32, 125, 10, 125]...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:118]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00038c420\u003e: \n    failed creating external auth \"ea-list\" in resourcegroup=\"ea-list-9lhb8x\" for cluster=\"ea-listgv59sm\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-9lhb8x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listgv59sm/externalAuths/ea-list\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: LockContention\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LockContention\",\n        \"message\": \"Failed to acquire lock: timed out\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed creating external auth \\\"ea-list\\\" in resourcegroup=\\\"ea-list-9lhb8x\\\" for cluster=\\\"ea-listgv59sm\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-9lhb8x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listgv59sm/externalAuths/ea-list\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"LockContention\\\",\\n    \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc00076e300\u003e{\n            ErrorCode: \"LockContention\",\n            StatusCode: 503,\n            RawResponse: {\n                Status: \"503 Service Unavailable\",\n                StatusCode: 503,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Retry-After\": [\"10\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/4616b9c4-eae5-42c0-b42e-e60136e4b27a\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Date\": [\n                        \"Thu, 12 Mar 2026 22:53:06 GMT\",\n                    ],\n                    \"Content-Length\": [\"186\"],\n                    \"X-Ms-Request-Id\": [\n                        \"0daaf6b9-dea5-419a-a781-f5d3f0c88c48\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Error-Code\": [\"LockContention\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: DD91E82BD13A4C8F974F7F6335BA4D4F Ref B: CHI261140506025 Ref C: 2026-03-12T22:52:57Z\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"10007\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"132f0482-3ab7-4255-98f8-45a73cf0c24c\",\n                    ],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260312T225307Z:132f0482-3ab7-4255-98f8-45a73cf0c24c\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc00038c200\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 76, 111, 99, 107, 67, 111, 110, 116, 101, 110, 116, 105, 111, 110, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 32, 116, 111, 32, 97, 99, 113, 117, 105, 114, 101, 32, 108, 111, 99, 107, 58, 32, 116, 105, 109, 101, 100, 32, 111, 117, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 54, 52, 102, 48, 54, 49, 57, 102, 45, 101, 98, 99, 50, 45, 52, 49, 53, 54, 45, 57, 100, 57, 49, 45, 99, 52, 99, 55, 56, 49, 100, 101, 55, 101, 53, 52, 34, 10, 32, 32, 32, 32, 125, 10, 125]...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:145]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00148e400\u003e: \n    failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-h2fml7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-h2fml7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: LockContention\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LockContention\",\n        \"message\": \"Failed to acquire lock: timed out\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create NodePool np-1: failed starting nodepool creation \\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-h2fml7\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-h2fml7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"LockContention\\\",\\n    \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00148e3e0\u003e{\n            msg: \"failed starting nodepool creation \\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-h2fml7\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-h2fml7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"LockContention\\\",\\n    \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000b78690\u003e{\n                ErrorCode: \"LockContention\",\n                StatusCode: 503,\n                RawResponse: {\n                    Status: \"503 Service Unavailable\",\n                    StatusCode: 503,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"Content-Length\": [\"186\"],\n                        \"X-Ms-Error-Code\": [\"LockContention\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"10025\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/ab289601-a1be-4eea-baec-7844c0a021f7\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Date\": [\n                            \"Thu, 12 Mar 2026 22:53:06 GMT\",\n                        ],\n                        \"Content-Type\": [\"application/json\"],\n                        \"Retry-After\": [\"10\"],\n                        \"X-Ms-Failure-Cause\": [\"service\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 6E495E82722B47FAB217CA309B5EFC10 Ref B: CHI261140501060 Ref C: 2026-03-12T22:52:57Z\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Ms-Request-Id\": [\n                            \"c86fd44b-9640-4399-b837-438b4fc03302\",\n                        ],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"8be5b8ee-e739-4b49-a85a-b05296aaa904\",\n                        ],\n               ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:145]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00148e400\u003e: \n    failed to create NodePool np-1: failed starting nodepool creation \"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-h2fml7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-h2fml7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: LockContention\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LockContention\",\n        \"message\": \"Failed to acquire lock: timed out\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create NodePool np-1: failed starting nodepool creation \\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-h2fml7\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-h2fml7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"LockContention\\\",\\n    \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00148e3e0\u003e{\n            msg: \"failed starting nodepool creation \\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-h2fml7\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-h2fml7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"LockContention\\\",\\n    \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000b78690\u003e{\n                ErrorCode: \"LockContention\",\n                StatusCode: 503,\n                RawResponse: {\n                    Status: \"503 Service Unavailable\",\n                    StatusCode: 503,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"Content-Length\": [\"186\"],\n                        \"X-Ms-Error-Code\": [\"LockContention\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"10025\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/ab289601-a1be-4eea-baec-7844c0a021f7\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Date\": [\n                            \"Thu, 12 Mar 2026 22:53:06 GMT\",\n                        ],\n                        \"Content-Type\": [\"application/json\"],\n                        \"Retry-After\": [\"10\"],\n                        \"X-Ms-Failure-Cause\": [\"service\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 6E495E82722B47FAB217CA309B5EFC10 Ref B: CHI261140501060 Ref C: 2026-03-12T22:52:57Z\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Ms-Request-Id\": [\n                            \"c86fd44b-9640-4399-b837-438b4fc03302\",\n                        ],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"8be5b8ee-e739-4b49-a85a-b05296aaa904\",\n                        ],\n               ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n    \u003c*errors.joinError | 0xc0015420c0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-pqj6m6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: LockContention\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LockContention\",\n        \"message\": \"Failed to acquire lock: timed out\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000840400\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-pqj6m6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"LockContention\\\",\\n    \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                err: \u003c*fmt.wrapError | 0xc0008403e0\u003e{\n                    msg: \"at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-pqj6m6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"LockContention\\\",\\n    \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                    err: \u003c*exported.ResponseError | 0xc0013a6450\u003e{\n                        ErrorCode: \"LockContention\",\n                        StatusCode: 503,\n                        RawResponse: {\n                            Status: \"503 Service Unavailable\",\n                            StatusCode: 503,\n                            Proto: \"HTTP/2.0\",\n                            ProtoMajor: 2,\n                            ProtoMinor: 0,\n                            Header: {\n                                \"Cache-Control\": [\"no-cache\"],\n                                \"Content-Length\": [\"186\"],\n                                \"X-Ms-Error-Code\": [\"LockContention\"],\n                                \"X-Ms-Request-Id\": [\n                                    \"57007d8e-6281-4f79-87e3-6b62771677b7\",\n                                ],\n                                \"X-Ms-Failure-Cause\": [\"service\"],\n                                \"X-Ms-Ratelimit-Remaining-Subscription-Global-Deletes\": [\"2999\"],\n                                \"X-Content-Type-Options\": [\"nosniff\"],\n                                \"Date\": [\n                                    \"Thu, 12 Mar 2026 20:29:07 GMT\",\n                                ],\n                                \"Pragma\": [\"no-cache\"],\n                                \"Content-Type\": [\"application/json\"],\n                                \"X-Envoy-Upstream-Service-Time\": [\"10009\"],\n                                \"X-Ms-Operation-Identifier\": [\n                                    \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/9a19b057-0917-4393-a2cf-ff9874abdb4b\",\n                                ],\n                                \"X-Ms-Correlation-Request-Id\": [\n                                    \"cd3510ea-1cec-45db-8f3f-4cac3c6e7c36\",\n                                ],\n                                \"Strict-Transport-Security\": [\n                                    \"max-age=31536000; includeSubDomains\",\n                                ],\n                                \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                                \"X-Msedge-Ref\": [\n                                    \"Ref A: DC15C07BB4844226852ED55513D0D6F1 Ref B: CHI261140507054 Re...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n    \u003c*errors.joinError | 0xc0015420c0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-pqj6m6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: LockContention\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LockContention\",\n        \"message\": \"Failed to acquire lock: timed out\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc000840400\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-pqj6m6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"LockContention\\\",\\n    \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                err: \u003c*fmt.wrapError | 0xc0008403e0\u003e{\n                    msg: \"at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-pqj6m6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"LockContention\\\",\\n    \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                    err: \u003c*exported.ResponseError | 0xc0013a6450\u003e{\n                        ErrorCode: \"LockContention\",\n                        StatusCode: 503,\n                        RawResponse: {\n                            Status: \"503 Service Unavailable\",\n                            StatusCode: 503,\n                            Proto: \"HTTP/2.0\",\n                            ProtoMajor: 2,\n                            ProtoMinor: 0,\n                            Header: {\n                                \"Cache-Control\": [\"no-cache\"],\n                                \"Content-Length\": [\"186\"],\n                                \"X-Ms-Error-Code\": [\"LockContention\"],\n                                \"X-Ms-Request-Id\": [\n                                    \"57007d8e-6281-4f79-87e3-6b62771677b7\",\n                                ],\n                                \"X-Ms-Failure-Cause\": [\"service\"],\n                                \"X-Ms-Ratelimit-Remaining-Subscription-Global-Deletes\": [\"2999\"],\n                                \"X-Content-Type-Options\": [\"nosniff\"],\n                                \"Date\": [\n                                    \"Thu, 12 Mar 2026 20:29:07 GMT\",\n                                ],\n                                \"Pragma\": [\"no-cache\"],\n                                \"Content-Type\": [\"application/json\"],\n                                \"X-Envoy-Upstream-Service-Time\": [\"10009\"],\n                                \"X-Ms-Operation-Identifier\": [\n                                    \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/9a19b057-0917-4393-a2cf-ff9874abdb4b\",\n                                ],\n                                \"X-Ms-Correlation-Request-Id\": [\n                                    \"cd3510ea-1cec-45db-8f3f-4cac3c6e7c36\",\n                                ],\n                                \"Strict-Transport-Security\": [\n                                    \"max-age=31536000; includeSubDomains\",\n                                ],\n                                \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                                \"X-Msedge-Ref\": [\n                                    \"Ref A: DC15C07BB4844226852ED55513D0D6F1 Ref B: CHI261140507054 Re...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032218705995239424","occurred_at":"2026-03-12T22:14:30Z","signature_id":"388b2629660a6fde682f686fb4f7b2082202b8c728cb670b1051b294c7126c5f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032218705995239424","occurred_at":"2026-03-12T22:14:30Z","signature_id":"bce8204f5375b3d33273c5e2620cd19a19f0689f319cc860948e1814d81110bc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032169081666277376","occurred_at":"2026-03-12T18:57:19Z","signature_id":"140bef2475018868d66f6328e7591dcba47de01ca6a279fe2b01cd85cf666493","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2031326084175237120","occurred_at":"2026-03-10T11:07:33Z","signature_id":"b817ba27d71b8cea588ac20c60c63b2f04beb1e330398a79f86b8a7e3f786ad4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"e519f1887d9740d59629140c909a7511daaa51821ca3498a363bc02241e26c7f","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-htfxlh\" in resourcegroup=\"admin-credential-lifecycle-test-qq776z\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-htfxlh in resource group admin-credential-lifecycle-test-qq776z, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":4.3478260869565215,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,1,0,0,2,0,0,0],"trend_range":"2026-03-08..2026-03-15 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00154e1e0\u003e: \n    failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-hb8wm7\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-hb8wm7, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-hb8wm7\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-hb8wm7, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001533930\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-hb8wm7\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00154e1e0\u003e: \n    failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-hb8wm7\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-hb8wm7, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-hb8wm7\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-hb8wm7, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001533930\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-hb8wm7\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:103]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001502510\u003e: \n    failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-7rvs2t\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-7rvs2t, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-7rvs2t\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-7rvs2t, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000879850\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-7rvs2t\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:103]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001502510\u003e: \n    failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-7rvs2t\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-7rvs2t, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-7rvs2t\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-7rvs2t, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000879850\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-7rvs2t\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:173]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00133e300\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-htfxlh\" in resourcegroup=\"admin-credential-lifecycle-test-qq776z\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-htfxlh in resource group admin-credential-lifecycle-test-qq776z, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-htfxlh\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-qq776z\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-htfxlh in resource group admin-credential-lifecycle-test-qq776z, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00135d130\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-htfxlh in resource group admin-credential-lifecycle-test-qq776z\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:173]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00133e300\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-htfxlh\" in resourcegroup=\"admin-credential-lifecycle-test-qq776z\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-htfxlh in resource group admin-credential-lifecycle-test-qq776z, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-htfxlh\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-qq776z\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-htfxlh in resource group admin-credential-lifecycle-test-qq776z, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00135d130\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-htfxlh in resource group admin-credential-lifecycle-test-qq776z\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2031998313082916864","occurred_at":"2026-03-12T07:38:45Z","signature_id":"a8ceb9bcd5d616af9202fd97fc9fb373bcebf9677610d7cd07e38d8adcf8020f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2031998313082916864","occurred_at":"2026-03-12T07:38:45Z","signature_id":"fa1d696f8e4e813800804eef5069426a7bb9b83fbda42396ee5a23a33eae492c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2031146014437543936","occurred_at":"2026-03-09T23:12:01Z","signature_id":"bf75d6986ad8d3bdbdcacceb45c276e6249f4fc2204ae216b864eb3317eb4d66","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"701a8a455e7ebdb1a55d6d89826d827a72387557689423a7e3811708950aebaa","failure_pattern":"ERROR CODE: InvalidTemplate; detail message Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'tags'. The parameters defined in the template are: 'clusterName, identities, openshiftControlPlaneVersionId, openshiftNodePoolVersionId, persistTagValue, usePooledIdentities'.","search_query":"ERROR CODE: InvalidTemplate","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes vm type NC4asT4v3 in a single cluster","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":4.3478260869565215,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,2,0,0,0,0,0],"trend_range":"2026-03-08..2026-03-15 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes vm type NC4asT4v3 in a single cluster","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:100]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000848600\u003e: \n    failed creating deployment \"aro-hcp-demo\" in resourcegroup=\"gpu-nodepools-NC4asT4v3-zhtprd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/gpu-nodepools-NC4asT4v3-zhtprd/providers/Microsoft.Resources/deployments/aro-hcp-demo\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplate\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplate\",\n        \"message\": \"Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'tags'. The parameters defined in the template are: 'clusterName, identities, openshiftControlPlaneVersionId, openshiftNodePoolVersionId, persistTagValue, usePooledIdentities'. Please see https://aka.ms/arm-pass-parameter-values for usage details.'.\",\n        \"additionalInfo\": [\n          {\n            \"type\": \"TemplateViolation\",\n            \"info\": {\n              \"lineNumber\": 0,\n              \"linePosition\": 0,\n              \"path\": \"\"\n            }\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed creating deployment \\\"aro-hcp-demo\\\" in resourcegroup=\\\"gpu-nodepools-NC4asT4v3-zhtprd\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/gpu-nodepools-NC4asT4v3-zhtprd/providers/Microsoft.Resources/deployments/aro-hcp-demo\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidTemplate\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidTemplate\\\",\\n    \\\"message\\\": \\\"Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'tags'. The parameters defined in the template are: 'clusterName, identities, openshiftControlPlaneVersionId, openshiftNodePoolVersionId, persistTagValue, usePooledIdentities'. Please see https://aka.ms/arm-pass-parameter-values for usage details.'.\\\",\\n    \\\"additionalInfo\\\": [\\n      {\\n        \\\"type\\\": \\\"TemplateViolation\\\",\\n        \\\"info\\\": {\\n          \\\"lineNumber\\\": 0,\\n          \\\"linePosition\\\": 0,\\n          \\\"path\\\": \\\"\\\"\\n        }\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc00135a990\u003e{\n            ErrorCode: \"InvalidTemplate\",\n            StatusCode: 400,\n            RawResponse: {\n                Status: \"400 Bad Request\",\n                StatusCode: 400,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Pragma\": [\"no-cache\"],\n                    \"X-Ms-Failure-Cause\": [\"gateway\"],\n                    \"X-Ms-Request-Id\": [\n                        \"f4d69cdb-d05b-44a9-9a92-6eddb4d52e6e\",\n                    ],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260310T160951Z:f4d69cdb-d05b-44a9-9a92-6eddb4d52e6e\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Content-Type\": [\n                        \"application/json; charset=utf-8\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"f4d69cdb-d05b-44a9-9a92-6eddb4d52e6e\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Date\": [\n                        \"Tue, 10 Mar 2026 16:09:50 GMT\",\n                    ],\n                    \"Content-Length\": [\"542\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 3F20D18F061F48B0A5963AE25F0A156A Ref B: BL2AA2011003040 Ref C: 2026-03-10T16:09:50Z\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc000848540\u003e{\n                    s: \"{\\\"error\\\":{\\\"code\\\":\\\"InvalidTemplate\\\",\\\"message\\\":\\\"Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'tags'. The parameters defined in the template are: 'clusterName, identities, openshiftControlPlaneVersionId, openshiftNodePoolVersionId, persistTagValue, usePooledIdentities'. Please see https://aka.ms/arm-pass-parameter-values for usage details.'.\\\",\\\"additionalInfo\\\":[{\\\"type\\\":\\\"TemplateViolation\\\",\\\"info\\\":{\\\"lineNumber\\\":0,\\\"linePosition\\\":0,\\\"path\\\":\\\"\\\"}}]}}\",\n                    i: 0,\n                },\n                ContentLength: 542,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompress...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:100]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000848600\u003e: \n    failed creating deployment \"aro-hcp-demo\" in resourcegroup=\"gpu-nodepools-NC4asT4v3-zhtprd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/gpu-nodepools-NC4asT4v3-zhtprd/providers/Microsoft.Resources/deployments/aro-hcp-demo\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplate\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplate\",\n        \"message\": \"Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'tags'. The parameters defined in the template are: 'clusterName, identities, openshiftControlPlaneVersionId, openshiftNodePoolVersionId, persistTagValue, usePooledIdentities'. Please see https://aka.ms/arm-pass-parameter-values for usage details.'.\",\n        \"additionalInfo\": [\n          {\n            \"type\": \"TemplateViolation\",\n            \"info\": {\n              \"lineNumber\": 0,\n              \"linePosition\": 0,\n              \"path\": \"\"\n            }\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed creating deployment \\\"aro-hcp-demo\\\" in resourcegroup=\\\"gpu-nodepools-NC4asT4v3-zhtprd\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/gpu-nodepools-NC4asT4v3-zhtprd/providers/Microsoft.Resources/deployments/aro-hcp-demo\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidTemplate\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidTemplate\\\",\\n    \\\"message\\\": \\\"Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'tags'. The parameters defined in the template are: 'clusterName, identities, openshiftControlPlaneVersionId, openshiftNodePoolVersionId, persistTagValue, usePooledIdentities'. Please see https://aka.ms/arm-pass-parameter-values for usage details.'.\\\",\\n    \\\"additionalInfo\\\": [\\n      {\\n        \\\"type\\\": \\\"TemplateViolation\\\",\\n        \\\"info\\\": {\\n          \\\"lineNumber\\\": 0,\\n          \\\"linePosition\\\": 0,\\n          \\\"path\\\": \\\"\\\"\\n        }\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc00135a990\u003e{\n            ErrorCode: \"InvalidTemplate\",\n            StatusCode: 400,\n            RawResponse: {\n                Status: \"400 Bad Request\",\n                StatusCode: 400,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Pragma\": [\"no-cache\"],\n                    \"X-Ms-Failure-Cause\": [\"gateway\"],\n                    \"X-Ms-Request-Id\": [\n                        \"f4d69cdb-d05b-44a9-9a92-6eddb4d52e6e\",\n                    ],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260310T160951Z:f4d69cdb-d05b-44a9-9a92-6eddb4d52e6e\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Content-Type\": [\n                        \"application/json; charset=utf-8\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"f4d69cdb-d05b-44a9-9a92-6eddb4d52e6e\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Date\": [\n                        \"Tue, 10 Mar 2026 16:09:50 GMT\",\n                    ],\n                    \"Content-Length\": [\"542\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 3F20D18F061F48B0A5963AE25F0A156A Ref B: BL2AA2011003040 Ref C: 2026-03-10T16:09:50Z\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc000848540\u003e{\n                    s: \"{\\\"error\\\":{\\\"code\\\":\\\"InvalidTemplate\\\",\\\"message\\\":\\\"Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'tags'. The parameters defined in the template are: 'clusterName, identities, openshiftControlPlaneVersionId, openshiftNodePoolVersionId, persistTagValue, usePooledIdentities'. Please see https://aka.ms/arm-pass-parameter-values for usage details.'.\\\",\\\"additionalInfo\\\":[{\\\"type\\\":\\\"TemplateViolation\\\",\\\"info\\\":{\\\"lineNumber\\\":0,\\\"linePosition\\\":0,\\\"path\\\":\\\"\\\"}}]}}\",\n                    i: 0,\n                },\n                ContentLength: 542,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompress...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:100]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0011369c0\u003e: \n    failed creating deployment \"aro-hcp-demo\" in resourcegroup=\"gpu-nodepools-NC4asT4v3-6t29q4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/gpu-nodepools-NC4asT4v3-6t29q4/providers/Microsoft.Resources/deployments/aro-hcp-demo\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplate\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplate\",\n        \"message\": \"Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'tags'. The parameters defined in the template are: 'clusterName, identities, openshiftControlPlaneVersionId, openshiftNodePoolVersionId, persistTagValue, usePooledIdentities'. Please see https://aka.ms/arm-pass-parameter-values for usage details.'.\",\n        \"additionalInfo\": [\n          {\n            \"type\": \"TemplateViolation\",\n            \"info\": {\n              \"lineNumber\": 0,\n              \"linePosition\": 0,\n              \"path\": \"\"\n            }\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed creating deployment \\\"aro-hcp-demo\\\" in resourcegroup=\\\"gpu-nodepools-NC4asT4v3-6t29q4\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/gpu-nodepools-NC4asT4v3-6t29q4/providers/Microsoft.Resources/deployments/aro-hcp-demo\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidTemplate\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidTemplate\\\",\\n    \\\"message\\\": \\\"Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'tags'. The parameters defined in the template are: 'clusterName, identities, openshiftControlPlaneVersionId, openshiftNodePoolVersionId, persistTagValue, usePooledIdentities'. Please see https://aka.ms/arm-pass-parameter-values for usage details.'.\\\",\\n    \\\"additionalInfo\\\": [\\n      {\\n        \\\"type\\\": \\\"TemplateViolation\\\",\\n        \\\"info\\\": {\\n          \\\"lineNumber\\\": 0,\\n          \\\"linePosition\\\": 0,\\n          \\\"path\\\": \\\"\\\"\\n        }\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc000eb44e0\u003e{\n            ErrorCode: \"InvalidTemplate\",\n            StatusCode: 400,\n            RawResponse: {\n                Status: \"400 Bad Request\",\n                StatusCode: 400,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Ms-Failure-Cause\": [\"gateway\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"6f0859fb-712e-493d-b9f3-1feb834d30d5\",\n                    ],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 0385E8E79B674F5DB2E6A6AC40C9C59D Ref B: CHI261140506042 Ref C: 2026-03-10T11:15:34Z\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Length\": [\"542\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Request-Id\": [\n                        \"6f0859fb-712e-493d-b9f3-1feb834d30d5\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Date\": [\n                        \"Tue, 10 Mar 2026 11:15:34 GMT\",\n                    ],\n                    \"Content-Type\": [\n                        \"application/json; charset=utf-8\",\n                    ],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260310T111535Z:6f0859fb-712e-493d-b9f3-1feb834d30d5\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc001136900\u003e{\n                    s: \"{\\\"error\\\":{\\\"code\\\":\\\"InvalidTemplate\\\",\\\"message\\\":\\\"Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'tags'. The parameters defined in the template are: 'clusterName, identities, openshiftControlPlaneVersionId, openshiftNodePoolVersionId, persistTagValue, usePooledIdentities'. Please see https://aka.ms/arm-pass-parameter-values for usage details.'.\\\",\\\"additionalInfo\\\":[{\\\"type\\\":\\\"TemplateViolation\\\",\\\"info\\\":{\\\"lineNumber\\\":0,\\\"linePosition\\\":0,\\\"path\\\":\\\"\\\"}}]}}\",\n                    i: 0,\n                },\n                ContentLength: 542,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompress...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:100]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0011369c0\u003e: \n    failed creating deployment \"aro-hcp-demo\" in resourcegroup=\"gpu-nodepools-NC4asT4v3-6t29q4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/gpu-nodepools-NC4asT4v3-6t29q4/providers/Microsoft.Resources/deployments/aro-hcp-demo\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplate\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplate\",\n        \"message\": \"Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'tags'. The parameters defined in the template are: 'clusterName, identities, openshiftControlPlaneVersionId, openshiftNodePoolVersionId, persistTagValue, usePooledIdentities'. Please see https://aka.ms/arm-pass-parameter-values for usage details.'.\",\n        \"additionalInfo\": [\n          {\n            \"type\": \"TemplateViolation\",\n            \"info\": {\n              \"lineNumber\": 0,\n              \"linePosition\": 0,\n              \"path\": \"\"\n            }\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed creating deployment \\\"aro-hcp-demo\\\" in resourcegroup=\\\"gpu-nodepools-NC4asT4v3-6t29q4\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/gpu-nodepools-NC4asT4v3-6t29q4/providers/Microsoft.Resources/deployments/aro-hcp-demo\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidTemplate\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidTemplate\\\",\\n    \\\"message\\\": \\\"Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'tags'. The parameters defined in the template are: 'clusterName, identities, openshiftControlPlaneVersionId, openshiftNodePoolVersionId, persistTagValue, usePooledIdentities'. Please see https://aka.ms/arm-pass-parameter-values for usage details.'.\\\",\\n    \\\"additionalInfo\\\": [\\n      {\\n        \\\"type\\\": \\\"TemplateViolation\\\",\\n        \\\"info\\\": {\\n          \\\"lineNumber\\\": 0,\\n          \\\"linePosition\\\": 0,\\n          \\\"path\\\": \\\"\\\"\\n        }\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc000eb44e0\u003e{\n            ErrorCode: \"InvalidTemplate\",\n            StatusCode: 400,\n            RawResponse: {\n                Status: \"400 Bad Request\",\n                StatusCode: 400,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Ms-Failure-Cause\": [\"gateway\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"6f0859fb-712e-493d-b9f3-1feb834d30d5\",\n                    ],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 0385E8E79B674F5DB2E6A6AC40C9C59D Ref B: CHI261140506042 Ref C: 2026-03-10T11:15:34Z\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Length\": [\"542\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Request-Id\": [\n                        \"6f0859fb-712e-493d-b9f3-1feb834d30d5\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Date\": [\n                        \"Tue, 10 Mar 2026 11:15:34 GMT\",\n                    ],\n                    \"Content-Type\": [\n                        \"application/json; charset=utf-8\",\n                    ],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260310T111535Z:6f0859fb-712e-493d-b9f3-1feb834d30d5\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc001136900\u003e{\n                    s: \"{\\\"error\\\":{\\\"code\\\":\\\"InvalidTemplate\\\",\\\"message\\\":\\\"Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'tags'. The parameters defined in the template are: 'clusterName, identities, openshiftControlPlaneVersionId, openshiftNodePoolVersionId, persistTagValue, usePooledIdentities'. Please see https://aka.ms/arm-pass-parameter-values for usage details.'.\\\",\\\"additionalInfo\\\":[{\\\"type\\\":\\\"TemplateViolation\\\",\\\"info\\\":{\\\"lineNumber\\\":0,\\\"linePosition\\\":0,\\\"path\\\":\\\"\\\"}}]}}\",\n                    i: 0,\n                },\n                ContentLength: 542,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompress...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2031390723315601408","occurred_at":"2026-03-10T15:24:24Z","signature_id":"6c441883c78820ae3841649469ac3aa331552f64ebfda94c0fb61f6a126777de","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2031326084175237120","occurred_at":"2026-03-10T11:07:33Z","signature_id":"e25997d0bada37ac5936ab51199b1aecc9df315e3d30cca7a3241f4e783e15c7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"18d5af65735558751d0c56d3c20458f218b143c5bf541874b01f142b2c1ce20c","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade using the candidate channel","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":4.3478260869565215,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,2,0,0,0],"trend_range":"2026-03-08..2026-03-15 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade using the candidate channel","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: Timed out after 2400.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001069d10\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \"4.20.15\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc00052cbc0\u003e{\n                msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n                err: \u003c*errors.errorString | 0xc0008a6470\u003e{\n                    s: \"no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n                },\n            },\n        ],\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: Timed out after 2400.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001069d10\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \"4.20.15\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc00052cbc0\u003e{\n                msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n                err: \u003c*errors.errorString | 0xc0008a6470\u003e{\n                    s: \"no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n                },\n            },\n        ],\n    }","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: Timed out after 2400.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f034d0\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \"4.20.15\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc00051c6c0\u003e{\n                msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n                err: \u003c*errors.errorString | 0xc00073e8e0\u003e{\n                    s: \"no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n                },\n            },\n        ],\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:109]: Timed out after 2400.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f034d0\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \"4.20.15\"\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc00051c6c0\u003e{\n                msg: \"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.15) failed: no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n                err: \u003c*errors.errorString | 0xc00073e8e0\u003e{\n                    s: \"no version in clusterversion/version status.history is greater than initial \\\"4.20.15\\\"\",\n                },\n            },\n        ],\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032218705995239424","occurred_at":"2026-03-12T22:14:30Z","signature_id":"b151282d3ca5fd5495336a240a79e07af4bf8b78005191d9a2f5b9e21ef10848","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032203554982203392","occurred_at":"2026-03-12T21:14:18Z","signature_id":"4891f73c10790637ee3b59dfd093b431ae5a5be9a59036add79bb898c720610d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"fcca862eb27e23509d85f6fcb1ad2960de66de78a777ff1696b296d9e75ced9d","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.1739130434782608,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,2,0,0,0,0],"trend_range":"2026-03-08..2026-03-15 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000c720a0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-qkwf4p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b05f658-bbb6-452d-b845-92443ec2c619\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b05f658-bbb6-452d-b845-92443ec2c619\",\n      \"name\": \"0b05f658-bbb6-452d-b845-92443ec2c619\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-11T03:36:35.705780329Z\",\n      \"endTime\": \"2026-03-11T03:38:07.574131728Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Network/locations/uksouth/operations/de3f3e34-ee55-4c5b-bdc7-7cbf3170faf2\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE UNAVAILABLE\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"message\\\": \\\"Operation de3f3e34-ee55-4c5b-bdc7-7cbf3170faf2 not found.\\\",\\n    \\\"details\\\": []\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-qkwf4p\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b05f658-bbb6-452d-b845-92443ec2c619\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b05f658-bbb6-452d-b845-92443ec2c619\\\",\\n  \\\"name\\\": \\\"0b05f658-bbb6-452d-b845-92443ec2c619\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-11T03:36:35.705780329Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-11T03:38:07.574131728Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Network/locations/uksouth/operations/de3f3e34-ee55-4c5b-bdc7-7cbf3170faf2\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 404: 404 Not Found\\\\nERROR CODE UNAVAILABLE\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Operation de3f3e34-ee55-4c5b-bdc7-7cbf3170faf2 not found.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": []\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000c72080\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-qkwf4p\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b05f658-bbb6-452d-b845-92443ec2c619\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b05f658-bbb6-452d-b845-92443ec2c619\\\",\\n  \\\"name\\\": \\\"0b05f658-bbb6-452d-b845-92443ec2c619\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-11T03:36:35.705780329Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-11T03:38:07.574131728Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Network/locations/uksouth/operations/de3f3e34-ee55-4c5b-bdc7-7cbf3170faf2\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 404: 404 Not Found\\\\nERROR CODE UNAVAILABLE\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Operation de3f3e34-ee55-4c5b-bdc7-7cbf3170faf2 not found.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": []\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00144fd10\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000c720a0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-qkwf4p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b05f658-bbb6-452d-b845-92443ec2c619\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b05f658-bbb6-452d-b845-92443ec2c619\",\n      \"name\": \"0b05f658-bbb6-452d-b845-92443ec2c619\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-11T03:36:35.705780329Z\",\n      \"endTime\": \"2026-03-11T03:38:07.574131728Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Network/locations/uksouth/operations/de3f3e34-ee55-4c5b-bdc7-7cbf3170faf2\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE UNAVAILABLE\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"message\\\": \\\"Operation de3f3e34-ee55-4c5b-bdc7-7cbf3170faf2 not found.\\\",\\n    \\\"details\\\": []\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-qkwf4p\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b05f658-bbb6-452d-b845-92443ec2c619\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b05f658-bbb6-452d-b845-92443ec2c619\\\",\\n  \\\"name\\\": \\\"0b05f658-bbb6-452d-b845-92443ec2c619\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-11T03:36:35.705780329Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-11T03:38:07.574131728Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Network/locations/uksouth/operations/de3f3e34-ee55-4c5b-bdc7-7cbf3170faf2\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 404: 404 Not Found\\\\nERROR CODE UNAVAILABLE\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Operation de3f3e34-ee55-4c5b-bdc7-7cbf3170faf2 not found.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": []\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000c72080\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-qkwf4p\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b05f658-bbb6-452d-b845-92443ec2c619\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0b05f658-bbb6-452d-b845-92443ec2c619\\\",\\n  \\\"name\\\": \\\"0b05f658-bbb6-452d-b845-92443ec2c619\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-11T03:36:35.705780329Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-11T03:38:07.574131728Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Network/locations/uksouth/operations/de3f3e34-ee55-4c5b-bdc7-7cbf3170faf2\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 404: 404 Not Found\\\\nERROR CODE UNAVAILABLE\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Operation de3f3e34-ee55-4c5b-bdc7-7cbf3170faf2 not found.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": []\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00144fd10\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0000d9940\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-6h5tqt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5c49f2a2-a1d3-45dc-b6e8-265bbbe3eadb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5c49f2a2-a1d3-45dc-b6e8-265bbbe3eadb\",\n      \"name\": \"5c49f2a2-a1d3-45dc-b6e8-265bbbe3eadb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-11T03:36:37.014555279Z\",\n      \"endTime\": \"2026-03-11T03:38:07.990052924Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Network/locations/uksouth/operations/bcedbd48-06f7-4d82-a965-ffd4f43ad6ae\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE UNAVAILABLE\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"message\\\": \\\"Operation bcedbd48-06f7-4d82-a965-ffd4f43ad6ae not found.\\\",\\n    \\\"details\\\": []\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-6h5tqt\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5c49f2a2-a1d3-45dc-b6e8-265bbbe3eadb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5c49f2a2-a1d3-45dc-b6e8-265bbbe3eadb\\\",\\n  \\\"name\\\": \\\"5c49f2a2-a1d3-45dc-b6e8-265bbbe3eadb\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-11T03:36:37.014555279Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-11T03:38:07.990052924Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Network/locations/uksouth/operations/bcedbd48-06f7-4d82-a965-ffd4f43ad6ae\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 404: 404 Not Found\\\\nERROR CODE UNAVAILABLE\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Operation bcedbd48-06f7-4d82-a965-ffd4f43ad6ae not found.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": []\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0000d9900\u003e{\n            msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-6h5tqt\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5c49f2a2-a1d3-45dc-b6e8-265bbbe3eadb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5c49f2a2-a1d3-45dc-b6e8-265bbbe3eadb\\\",\\n  \\\"name\\\": \\\"5c49f2a2-a1d3-45dc-b6e8-265bbbe3eadb\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-11T03:36:37.014555279Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-11T03:38:07.990052924Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Network/locations/uksouth/operations/bcedbd48-06f7-4d82-a965-ffd4f43ad6ae\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 404: 404 Not Found\\\\nERROR CODE UNAVAILABLE\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Operation bcedbd48-06f7-4d82-a965-ffd4f43ad6ae not found.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": []\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000f006c0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Routing-Request-Id\": [\n       ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0000d9940\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-6h5tqt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5c49f2a2-a1d3-45dc-b6e8-265bbbe3eadb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5c49f2a2-a1d3-45dc-b6e8-265bbbe3eadb\",\n      \"name\": \"5c49f2a2-a1d3-45dc-b6e8-265bbbe3eadb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-11T03:36:37.014555279Z\",\n      \"endTime\": \"2026-03-11T03:38:07.990052924Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Network/locations/uksouth/operations/bcedbd48-06f7-4d82-a965-ffd4f43ad6ae\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE UNAVAILABLE\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"message\\\": \\\"Operation bcedbd48-06f7-4d82-a965-ffd4f43ad6ae not found.\\\",\\n    \\\"details\\\": []\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-6h5tqt\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5c49f2a2-a1d3-45dc-b6e8-265bbbe3eadb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5c49f2a2-a1d3-45dc-b6e8-265bbbe3eadb\\\",\\n  \\\"name\\\": \\\"5c49f2a2-a1d3-45dc-b6e8-265bbbe3eadb\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-11T03:36:37.014555279Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-11T03:38:07.990052924Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Network/locations/uksouth/operations/bcedbd48-06f7-4d82-a965-ffd4f43ad6ae\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 404: 404 Not Found\\\\nERROR CODE UNAVAILABLE\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Operation bcedbd48-06f7-4d82-a965-ffd4f43ad6ae not found.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": []\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0000d9900\u003e{\n            msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-6h5tqt\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5c49f2a2-a1d3-45dc-b6e8-265bbbe3eadb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5c49f2a2-a1d3-45dc-b6e8-265bbbe3eadb\\\",\\n  \\\"name\\\": \\\"5c49f2a2-a1d3-45dc-b6e8-265bbbe3eadb\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-11T03:36:37.014555279Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-11T03:38:07.990052924Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Network/locations/uksouth/operations/bcedbd48-06f7-4d82-a965-ffd4f43ad6ae\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 404: 404 Not Found\\\\nERROR CODE UNAVAILABLE\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n  \\\\\\\"error\\\\\\\": {\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Operation bcedbd48-06f7-4d82-a965-ffd4f43ad6ae not found.\\\\\\\",\\\\n    \\\\\\\"details\\\\\\\": []\\\\n  }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000f006c0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Routing-Request-Id\": [\n       ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2031571552545280000","occurred_at":"2026-03-11T03:22:57Z","signature_id":"7b7b054752b0885c9af570caad57b1f44f99fc1b96bb07352ce71135ad051b56","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2031571552545280000","occurred_at":"2026-03-11T03:22:57Z","signature_id":"cc70448829c646dd481f908571d950a4d5bc036908b8332bda15676a93a9de52","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"cedc818b7e720916031ed8d299fdf1177b8336a53376f99af83f86d0526a86bb","failure_pattern":"Admin credential 1 should fail after revocation, last error: \u003cnil\u003e","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.1739130434782608,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-03-08..2026-03-15 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:222]: Admin credential 1 should fail after revocation, last error: \u003cnil\u003e\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    {}\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:222]: Admin credential 1 should fail after revocation, last error: \u003cnil\u003e\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    {}\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032871090371432448","occurred_at":"2026-03-14T17:26:51Z","signature_id":"d617bc01711b19fec7ae259e6b1da87ceb0b303c46e17ba4e53b8c013178e381","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"e9065928ca1d323095a060fb425d3a601fe5817c1279001c85a6c30b326c861b","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.1739130434782608,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-03-08..2026-03-15 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:152]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:152]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2031571552545280000","occurred_at":"2026-03-11T03:22:57Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"e3d87f9711b73ee2df76304d88a8f550b1fc646bcbf24326d13ce9cee805cbde","failure_pattern":"ERROR CODE: Conflict; detail message Cannot create resource while parent resource is provisioning; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.1739130434782608,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-03-08..2026-03-15 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-02"],"prior_runs_affected":2,"prior_last_seen_at":"2026-03-06T00:41:27Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0007b0fa0\u003e: \n    failed to create NodePool arm64-vm-np-1: failed starting nodepool creation \"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-8h7xd7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-8h7xd7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster/nodePools/arm64-vm-np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot create resource while parent resource is provisioning\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-8h7xd7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster/nodePools/arm64-vm-np-1\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create NodePool arm64-vm-np-1: failed starting nodepool creation \\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-8h7xd7\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-8h7xd7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster/nodePools/arm64-vm-np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-8h7xd7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster/nodePools/arm64-vm-np-1\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0007b0f60\u003e{\n            msg: \"failed starting nodepool creation \\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-8h7xd7\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-8h7xd7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster/nodePools/arm64-vm-np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-8h7xd7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster/nodePools/arm64-vm-np-1\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0015e0990\u003e{\n                ErrorCode: \"Conflict\",\n                StatusCode: 409,\n                RawResponse: {\n                    Status: \"409 Conflict\",\n                    StatusCode: 409,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 1A43493D763743078F50D58A7107C67A Ref B: CHI261140511036 Ref C: 2026-03-09T15:39:54Z\",\n                        ],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                        \"Content-Length\": [\"348\"],\n                        \"X-Ms-Request-Id\": [\n                            \"b47e62cf-b184-4859-bc06-a546313b427d\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"UKSOUTH:20260309T153955Z:71f1c6be-0050-4562-ae53-b90c10721c0e\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"169\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"71f1c6be-0050-4562-ae53-b90c10721c0e\",\n                        ],\n                        \"Date\": [\n                            \"Mon, 09 Mar 2026 15:39:54 GMT\",\n                        ],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Error-Code\": [\"Conflict\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=64dc69e4-d083-49fc-9569-ebece...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0007b0fa0\u003e: \n    failed to create NodePool arm64-vm-np-1: failed starting nodepool creation \"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-8h7xd7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-8h7xd7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster/nodePools/arm64-vm-np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot create resource while parent resource is provisioning\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-8h7xd7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster/nodePools/arm64-vm-np-1\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create NodePool arm64-vm-np-1: failed starting nodepool creation \\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-8h7xd7\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-8h7xd7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster/nodePools/arm64-vm-np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-8h7xd7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster/nodePools/arm64-vm-np-1\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0007b0f60\u003e{\n            msg: \"failed starting nodepool creation \\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-8h7xd7\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-8h7xd7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster/nodePools/arm64-vm-np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Cannot create resource while parent resource is provisioning\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-8h7xd7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/arm64-vm-hcp-cluster/nodePools/arm64-vm-np-1\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0015e0990\u003e{\n                ErrorCode: \"Conflict\",\n                StatusCode: 409,\n                RawResponse: {\n                    Status: \"409 Conflict\",\n                    StatusCode: 409,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 1A43493D763743078F50D58A7107C67A Ref B: CHI261140511036 Ref C: 2026-03-09T15:39:54Z\",\n                        ],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                        \"Content-Length\": [\"348\"],\n                        \"X-Ms-Request-Id\": [\n                            \"b47e62cf-b184-4859-bc06-a546313b427d\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"UKSOUTH:20260309T153955Z:71f1c6be-0050-4562-ae53-b90c10721c0e\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"169\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"71f1c6be-0050-4562-ae53-b90c10721c0e\",\n                        ],\n                        \"Date\": [\n                            \"Mon, 09 Mar 2026 15:39:54 GMT\",\n                        ],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Error-Code\": [\"Conflict\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=64dc69e4-d083-49fc-9569-ebece...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2031028912292630528","occurred_at":"2026-03-09T15:26:41Z","signature_id":"3fe282016c061cf4a29995542af7308b93690adfe0fbba251de5fc2afec65bd0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"50c48c88fe4eb95e98afa85c61c4077e079f1b9e83d2b9321ba1c497a35a22ac","failure_pattern":"ERROR CODE: Conflict; detail message Resource is already deleting; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.1739130434782608,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-03-08..2026-03-15 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n    \u003c*errors.joinError | 0xc000d78048\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-sqdgm5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-vzqchl\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Resource is already deleting\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-sqdgm5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-vzqchl\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc001509a00\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-sqdgm5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-vzqchl\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Resource is already deleting\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-sqdgm5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-vzqchl\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                err: \u003c*fmt.wrapError | 0xc0015099e0\u003e{\n                    msg: \"at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-sqdgm5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-vzqchl\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Resource is already deleting\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-sqdgm5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-vzqchl\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                    err: \u003c*exported.ResponseError | 0xc0009cc510\u003e{\n                        ErrorCode: \"Conflict\",\n                        StatusCode: 409,\n                        RawResponse: {\n                            Status: \"409 Conflict\",\n                            StatusCode: 409,\n                            Proto: \"HTTP/2.0\",\n                            ProtoMajor: 2,\n                            ProtoMinor: 0,\n                            Header: {\n                                \"Content-Length\": [\"295\"],\n                                \"X-Ms-Ratelimit-Remaining-Subscription-Global-Deletes\": [\"3000\"],\n                                \"X-Ms-Correlation-Request-Id\": [\n                                    \"36e4392a-0d35-4147-8a9f-6300ad30a25c\",\n                                ],\n                                \"X-Ms-Routing-Request-Id\": [\n                                    \"UKSOUTH:20260309T050748Z:36e4392a-0d35-4147-8a9f-6300ad30a25c\",\n                                ],\n                                \"Cache-Control\": [\"no-cache\"],\n                                \"Expires\": [\"-1\"],\n                                \"X-Ms-Request-Id\": [\n                                    \"9240eef9-4396-4070-b424-2e847f902062\",\n                                ],\n                                \"X-Envoy-Upstream-Service-Time\": [\"32460\"],\n                                \"X-Ms-Ratelimit-Remaining-Subscription-Deletes\": [\"200\"],\n                                \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                                \"Date\": [\n                                    \"Mon, 09 Mar 2026 05:07:47 GMT\",\n                                ],\n                                \"Content-Type\": [\"application/json\"],\n                                \"X-Ms-Error-Code\": [\"Conflict\"],\n                                \"X-Ms-Operation-Identifier\": [\n                                    \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/1795cff2-640d-4155-ab23-dd61fa932a34\",\n                                ],\n  ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n    \u003c*errors.joinError | 0xc000d78048\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-sqdgm5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-vzqchl\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Resource is already deleting\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-sqdgm5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-vzqchl\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc001509a00\u003e{\n                msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-sqdgm5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-vzqchl\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Resource is already deleting\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-sqdgm5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-vzqchl\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                err: \u003c*fmt.wrapError | 0xc0015099e0\u003e{\n                    msg: \"at least one hcp cluster failed to delete: DELETE https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-sqdgm5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-vzqchl\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Resource is already deleting\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-sqdgm5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-vzqchl\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                    err: \u003c*exported.ResponseError | 0xc0009cc510\u003e{\n                        ErrorCode: \"Conflict\",\n                        StatusCode: 409,\n                        RawResponse: {\n                            Status: \"409 Conflict\",\n                            StatusCode: 409,\n                            Proto: \"HTTP/2.0\",\n                            ProtoMajor: 2,\n                            ProtoMinor: 0,\n                            Header: {\n                                \"Content-Length\": [\"295\"],\n                                \"X-Ms-Ratelimit-Remaining-Subscription-Global-Deletes\": [\"3000\"],\n                                \"X-Ms-Correlation-Request-Id\": [\n                                    \"36e4392a-0d35-4147-8a9f-6300ad30a25c\",\n                                ],\n                                \"X-Ms-Routing-Request-Id\": [\n                                    \"UKSOUTH:20260309T050748Z:36e4392a-0d35-4147-8a9f-6300ad30a25c\",\n                                ],\n                                \"Cache-Control\": [\"no-cache\"],\n                                \"Expires\": [\"-1\"],\n                                \"X-Ms-Request-Id\": [\n                                    \"9240eef9-4396-4070-b424-2e847f902062\",\n                                ],\n                                \"X-Envoy-Upstream-Service-Time\": [\"32460\"],\n                                \"X-Ms-Ratelimit-Remaining-Subscription-Deletes\": [\"200\"],\n                                \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                                \"Date\": [\n                                    \"Mon, 09 Mar 2026 05:07:47 GMT\",\n                                ],\n                                \"Content-Type\": [\"application/json\"],\n                                \"X-Ms-Error-Code\": [\"Conflict\"],\n                                \"X-Ms-Operation-Identifier\": [\n                                    \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/1795cff2-640d-4155-ab23-dd61fa932a34\",\n                                ],\n  ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2030862218051981312","occurred_at":"2026-03-09T04:24:18Z","signature_id":"f61ea001969b935f4cb0e6018734abf319697e1038a28d23d5f55c5786e845d2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"dc4ab62f3a343f11870a43df584c90b01513244ce0557845967def276b1e02cb","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InvalidRequestContent; detail message node pool \u003cnodepool\u003e '4.20.15' must not be greater than Control Plane version '4.20.8'","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes vm type NC4asT4v3 in a single cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.1739130434782608,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-03-08..2026-03-15 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes vm type NC4asT4v3 in a single cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:100]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000fc6740\u003e: \n    failed waiting for deployment \"aro-hcp-demo\" in resourcegroup=\"gpu-nodepools-NC4asT4v3-w6hmz9\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/gpu-nodepools-NC4asT4v3-w6hmz9/providers/Microsoft.Resources/deployments/aro-hcp-demo/operationStatuses/08584282515392752534\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/gpu-nodepools-NC4asT4v3-w6hmz9/providers/Microsoft.Resources/deployments/nodepool-1\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"InvalidRequestContent\\\",\\r\\n            \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed waiting for deployment \\\"aro-hcp-demo\\\" in resourcegroup=\\\"gpu-nodepools-NC4asT4v3-w6hmz9\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/gpu-nodepools-NC4asT4v3-w6hmz9/providers/Microsoft.Resources/deployments/aro-hcp-demo/operationStatuses/08584282515392752534\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/gpu-nodepools-NC4asT4v3-w6hmz9/providers/Microsoft.Resources/deployments/nodepool-1\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"InvalidRequestContent\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc001500e10\u003e{\n            ErrorCode: \"DeploymentFailed\",\n            StatusCode: 200,\n            RawResponse: {\n                Status: \"200 OK\",\n                StatusCode: 200,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Length\": [\"1243\"],\n                    \"X-Ms-Request-Id\": [\n                        \"0d231ad8-8318-4e3a-920c-9541e2affd70\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"0d231ad8-8318-4e3a-920c-9541e2affd70\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Content-Type\": [\n                        \"application/json; charset=utf-8\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3748\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260312T225347Z:0d231ad8-8318-4e3a-920c-9541e2affd70\",\n                    ],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 4733BC488DE8424898A45EF16542E708 Ref B: CH1AA2020603045 Ref C: 2026-03-12T22:53:47Z\",\n                    ],\n                    \"Date\": [\n                        \"Thu, 12 Mar 2026 22:53:46 GMT\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc000fc6620\u003e{\n                    s:...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:100]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000fc6740\u003e: \n    failed waiting for deployment \"aro-hcp-demo\" in resourcegroup=\"gpu-nodepools-NC4asT4v3-w6hmz9\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/gpu-nodepools-NC4asT4v3-w6hmz9/providers/Microsoft.Resources/deployments/aro-hcp-demo/operationStatuses/08584282515392752534\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/gpu-nodepools-NC4asT4v3-w6hmz9/providers/Microsoft.Resources/deployments/nodepool-1\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"InvalidRequestContent\\\",\\r\\n            \\\"message\\\": \\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed waiting for deployment \\\"aro-hcp-demo\\\" in resourcegroup=\\\"gpu-nodepools-NC4asT4v3-w6hmz9\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/gpu-nodepools-NC4asT4v3-w6hmz9/providers/Microsoft.Resources/deployments/aro-hcp-demo/operationStatuses/08584282515392752534\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/gpu-nodepools-NC4asT4v3-w6hmz9/providers/Microsoft.Resources/deployments/nodepool-1\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"InvalidRequestContent\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Node pool version '4.20.15' must not be greater than Control Plane version '4.20.8'\\\\\\\"\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc001500e10\u003e{\n            ErrorCode: \"DeploymentFailed\",\n            StatusCode: 200,\n            RawResponse: {\n                Status: \"200 OK\",\n                StatusCode: 200,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Length\": [\"1243\"],\n                    \"X-Ms-Request-Id\": [\n                        \"0d231ad8-8318-4e3a-920c-9541e2affd70\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"0d231ad8-8318-4e3a-920c-9541e2affd70\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Content-Type\": [\n                        \"application/json; charset=utf-8\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3748\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260312T225347Z:0d231ad8-8318-4e3a-920c-9541e2affd70\",\n                    ],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 4733BC488DE8424898A45EF16542E708 Ref B: CH1AA2020603045 Ref C: 2026-03-12T22:53:47Z\",\n                    ],\n                    \"Date\": [\n                        \"Thu, 12 Mar 2026 22:53:46 GMT\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc000fc6620\u003e{\n                    s:...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032218705995239424","occurred_at":"2026-03-12T22:14:30Z","signature_id":"1373e9545300a255cd138cd652149b1b3821540417659bab0182ceebf83220bc","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"d2973977438d8e970cb26c45be13f7a3ebd47e52a45ae782e685cf476877173a","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.1739130434782608,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-03-08..2026-03-15 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0012ecd00\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-9wh75v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-9wh75v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc00130de60\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Request-Id\": [\n                        \"aded1fbf-5095-4f14-957e-7e4810dfa7b7\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"1537\"],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/ffa8e515-a5e8-40aa-9ef9-9df55097710b\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260309T155315Z:1e0a595f-913b-42ed-b06c-d8a52395722d\",\n                    ],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: D4B3120337FD4E1A835EB7502426C822 Ref B: CH1AA2020603037 Ref C: 2026-03-09T15:53:13Z\",\n                    ],\n                    \"Date\": [\n                        \"Mon, 09 Mar 2026 15:53:14 GMT\",\n                    ],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"1e0a595f-913b-42ed-b06c-d8a52395722d\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0012ecc40\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        H...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0012ecd00\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-9wh75v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-9wh75v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc00130de60\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Request-Id\": [\n                        \"aded1fbf-5095-4f14-957e-7e4810dfa7b7\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"1537\"],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/ffa8e515-a5e8-40aa-9ef9-9df55097710b\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260309T155315Z:1e0a595f-913b-42ed-b06c-d8a52395722d\",\n                    ],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: D4B3120337FD4E1A835EB7502426C822 Ref B: CH1AA2020603037 Ref C: 2026-03-09T15:53:13Z\",\n                    ],\n                    \"Date\": [\n                        \"Mon, 09 Mar 2026 15:53:14 GMT\",\n                    ],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"1e0a595f-913b-42ed-b06c-d8a52395722d\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0012ecc40\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        H...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2031028912292630528","occurred_at":"2026-03-09T15:26:41Z","signature_id":"19f0e1d63b5ac8ac27542e5db6fbfc13fcc5a2136385381f80f4dad2fdbce086","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"ed91849277e23f84f2cfe3e0bb2a2379781cf9007095371a6220d51d0605b4b7","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.1739130434782608,"anchor_occurrences":1,"after_last_push_count":46,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-03-08..2026-03-15 UTC","prior_windows_present":2,"prior_window_starts":["2026-02-23","2026-03-02"],"prior_runs_affected":13,"prior_last_seen_at":"2026-03-04T09:08:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2031521952853659648","occurred_at":"2026-03-11T00:05:52Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"21585705694045faebee4379e2c73b35692dc962645ec40fec35028a087c9da4","failure_pattern":"Should receive valid HTTP status code from authorized IP","search_query":"Should receive valid HTTP status code from authorized IP","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.1739130434782608,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-03-08..2026-03-15 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:168]: Timed out after 120.002s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:167 with:\nShould receive valid HTTP status code from authorized IP\nExpected\n    \u003cstring\u003e: 000\nto match regular expression\n    \u003cstring\u003e: ^[2-5][0-9][0-9]$\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:168]: Timed out after 120.002s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:167 with:\nShould receive valid HTTP status code from authorized IP\nExpected\n    \u003cstring\u003e: 000\nto match regular expression\n    \u003cstring\u003e: ^[2-5][0-9][0-9]$"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032084631825682432","occurred_at":"2026-03-12T13:21:45Z","signature_id":"bb84802bfacd4ce780e314d30a326a7b5b7c16cac47e89a4a091ee0caceff220","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"083a9cceffc299c1feee488fbf6249c7387be0a488330c96c1fef09ca1fc3da6","failure_pattern":"Timed out after 120.345s.","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.1739130434782608,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-03-08..2026-03-15 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:168]: Timed out after 120.345s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:162 with:\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    {}\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:168]: Timed out after 120.345s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:162 with:\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    {}\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2032610884349595648","occurred_at":"2026-03-14T00:12:53Z","signature_id":"d88c4a00af6acde1a56adecafa1580decbd06e9145355918ea48ded5553fde30","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"e457d9ded8188e2e5d3bf94632039456ab0d3f40aab17c66a163827314f30476","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.1739130434782608,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-03-08..2026-03-15 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:222]: Admin credential 1 should fail after revocation, last error: client rate limiter Wait returned an error: context deadline exceeded\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    {}\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:222]: Admin credential 1 should fail after revocation, last error: client rate limiter Wait returned an error: context deadline exceeded\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    {}\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2033051021734842368","occurred_at":"2026-03-15T05:21:50Z","signature_id":"59c642fdc88f1fb11d46b08cbd20bafcf3e7f95194d6e3ce38cdbe2bc337ceb7","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}