{"meta":{"start_date":"2026-03-16","end_date":"2026-03-22","start_at":"2026-03-16T00:00:00Z","end_at":"2026-03-23T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T17:16:33Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":39,"failed_runs":20,"raw_occurrences":81,"matched_occurrences":80,"runs_affected":19},"rows":[{"environment":"prod","failure_pattern_id":"8a0eb43d82c81f5ac221f93e261d7d3906eccf9364890faef15f25844bdadb79","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","occurrences":18,"runs_affected":2,"failed_runs":2,"run_impact_percent":5.128205128205129,"anchor_occurrences":18,"after_last_push_count":23,"trend_counts":[0,7,0,11,0,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-02"],"prior_runs_affected":3,"prior_last_seen_at":"2026-03-04T11:18:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000962220\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-hfk97l: failed waiting for cluster=\"cluster-zstream-4-20-hfk97l\" in resourcegroup=\"rg-zstream-upgrade-4-20-6jjt78\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\",\n      \"name\": \"a535b9a7-79bf-4ae9-99b6-679319213e49\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:10:00.515143027Z\",\n      \"endTime\": \"2026-03-18T22:11:58.502121438Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-20-hfk97l: failed waiting for cluster=\\\"cluster-zstream-4-20-hfk97l\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-6jjt78\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\\\",\\n  \\\"name\\\": \\\"a535b9a7-79bf-4ae9-99b6-679319213e49\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:10:00.515143027Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:11:58.502121438Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0009621e0\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-hfk97l\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-6jjt78\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\\\",\\n  \\\"name\\\": \\\"a535b9a7-79bf-4ae9-99b6-679319213e49\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:10:00.515143027Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:11:58.502121438Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000e803c0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/a23e4d00-7c12-428e-b637-d6e87c428e63\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260318T221203Z:86a9a285-d268-4eaa-81f2-1b43b6865e4f\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"503\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"86a9a285-d268-4eaa-81f2-1b43b6865e4f\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 30110B94D9A9488595249B68A2A891EF Ref B: CHI261140505062 Ref C: 2026-03-18T22:12:03Z\",\n                        ],\n                        \"Date\": [\n                            \"Wed, 18 Mar 2026 22:12...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000962220\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-hfk97l: failed waiting for cluster=\"cluster-zstream-4-20-hfk97l\" in resourcegroup=\"rg-zstream-upgrade-4-20-6jjt78\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\",\n      \"name\": \"a535b9a7-79bf-4ae9-99b6-679319213e49\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:10:00.515143027Z\",\n      \"endTime\": \"2026-03-18T22:11:58.502121438Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-20-hfk97l: failed waiting for cluster=\\\"cluster-zstream-4-20-hfk97l\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-6jjt78\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\\\",\\n  \\\"name\\\": \\\"a535b9a7-79bf-4ae9-99b6-679319213e49\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:10:00.515143027Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:11:58.502121438Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0009621e0\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-hfk97l\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-6jjt78\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\\\",\\n  \\\"name\\\": \\\"a535b9a7-79bf-4ae9-99b6-679319213e49\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:10:00.515143027Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:11:58.502121438Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000e803c0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/a23e4d00-7c12-428e-b637-d6e87c428e63\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260318T221203Z:86a9a285-d268-4eaa-81f2-1b43b6865e4f\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"503\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"86a9a285-d268-4eaa-81f2-1b43b6865e4f\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 30110B94D9A9488595249B68A2A891EF Ref B: CHI261140505062 Ref C: 2026-03-18T22:12:03Z\",\n                        ],\n                        \"Date\": [\n                            \"Wed, 18 Mar 2026 22:12...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000a8e6a0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-fncbll\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\",\n      \"name\": \"1dcfdab9-27df-483f-a009-42303866394b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:10:00.698917174Z\",\n      \"endTime\": \"2026-03-18T22:12:23.691383966Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-fncbll\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\\\",\\n  \\\"name\\\": \\\"1dcfdab9-27df-483f-a009-42303866394b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:10:00.698917174Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:12:23.691383966Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000a8e680\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-fncbll\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\\\",\\n  \\\"name\\\": \\\"1dcfdab9-27df-483f-a009-42303866394b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:10:00.698917174Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:12:23.691383966Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00094b230\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Length\": [\"503\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Request-Id\": [\n                            \"ee86427e-005d-4cc8-bd01-2b82a0056536\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADAEAST:20260318T221233Z:a23b9d02-f439-4d75-bcef-bec5f70b2688\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 2529055C24EC4D90ABA73CA7E8A5F278 Ref B: CHI261140504054 Ref C: 2026-03-18T22:12:33Z\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"11\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"a23b9d02-f439-4d75-bcef-bec5f70b2688\",\n                        ],\n                        \"Strict-Transport-Security\": [\n     ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000a8e6a0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-fncbll\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\",\n      \"name\": \"1dcfdab9-27df-483f-a009-42303866394b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:10:00.698917174Z\",\n      \"endTime\": \"2026-03-18T22:12:23.691383966Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-fncbll\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\\\",\\n  \\\"name\\\": \\\"1dcfdab9-27df-483f-a009-42303866394b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:10:00.698917174Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:12:23.691383966Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000a8e680\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-fncbll\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\\\",\\n  \\\"name\\\": \\\"1dcfdab9-27df-483f-a009-42303866394b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:10:00.698917174Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:12:23.691383966Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00094b230\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Length\": [\"503\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Request-Id\": [\n                            \"ee86427e-005d-4cc8-bd01-2b82a0056536\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADAEAST:20260318T221233Z:a23b9d02-f439-4d75-bcef-bec5f70b2688\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 2529055C24EC4D90ABA73CA7E8A5F278 Ref B: CHI261140504054 Ref C: 2026-03-18T22:12:33Z\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"11\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"a23b9d02-f439-4d75-bcef-bec5f70b2688\",\n                        ],\n                        \"Strict-Transport-Security\": [\n     ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000b943a0\u003e: \n    failed to create HCP cluster ea-list8n6lb4: failed waiting for cluster=\"ea-list8n6lb4\" in resourcegroup=\"ea-list-gwfwm5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\",\n      \"name\": \"b5aa74e8-43d5-4cff-b520-63b6b9163130\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T21:58:12.305820102Z\",\n      \"endTime\": \"2026-03-18T21:59:35.853803496Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-list8n6lb4: failed waiting for cluster=\\\"ea-list8n6lb4\\\" in resourcegroup=\\\"ea-list-gwfwm5\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\\\",\\n  \\\"name\\\": \\\"b5aa74e8-43d5-4cff-b520-63b6b9163130\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T21:58:12.305820102Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T21:59:35.853803496Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000b94380\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-list8n6lb4\\\" in resourcegroup=\\\"ea-list-gwfwm5\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\\\",\\n  \\\"name\\\": \\\"b5aa74e8-43d5-4cff-b520-63b6b9163130\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T21:58:12.305820102Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T21:59:35.853803496Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0011866c0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260318T215943Z:3c196c67-a4c1-40eb-abc4-c5d8f5d91317\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Date\": [\n                            \"Wed, 18 Mar 2026 21:59:43 GMT\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Ms-Request-Id\": [\n                            \"e2863777-8b30-4013-8910-cf48bfc93c57\",\n                        ],\n                        \"X-Envoy-Upstream-Service-Time\": [\"11\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 694EB5D5CF094E458C2CC1B85768A697 Ref B: CHI261140510060 Ref C: 2026-03-18T21:59:43Z\",\n                        ],\n                        \"Content-Length\": [\"503\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Expires...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000b943a0\u003e: \n    failed to create HCP cluster ea-list8n6lb4: failed waiting for cluster=\"ea-list8n6lb4\" in resourcegroup=\"ea-list-gwfwm5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\",\n      \"name\": \"b5aa74e8-43d5-4cff-b520-63b6b9163130\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T21:58:12.305820102Z\",\n      \"endTime\": \"2026-03-18T21:59:35.853803496Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-list8n6lb4: failed waiting for cluster=\\\"ea-list8n6lb4\\\" in resourcegroup=\\\"ea-list-gwfwm5\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\\\",\\n  \\\"name\\\": \\\"b5aa74e8-43d5-4cff-b520-63b6b9163130\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T21:58:12.305820102Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T21:59:35.853803496Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000b94380\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-list8n6lb4\\\" in resourcegroup=\\\"ea-list-gwfwm5\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\\\",\\n  \\\"name\\\": \\\"b5aa74e8-43d5-4cff-b520-63b6b9163130\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T21:58:12.305820102Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T21:59:35.853803496Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0011866c0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260318T215943Z:3c196c67-a4c1-40eb-abc4-c5d8f5d91317\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Date\": [\n                            \"Wed, 18 Mar 2026 21:59:43 GMT\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Ms-Request-Id\": [\n                            \"e2863777-8b30-4013-8910-cf48bfc93c57\",\n                        ],\n                        \"X-Envoy-Upstream-Service-Time\": [\"11\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 694EB5D5CF094E458C2CC1B85768A697 Ref B: CHI261140510060 Ref C: 2026-03-18T21:59:43Z\",\n                        ],\n                        \"Content-Length\": [\"503\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Expires...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"06635d020ca291cd1c2ba88618a73ef47ef44b12a92afe822f14d0c83ccde9f6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"175c269dca6d1137eef269c582abddb7ea6d16d67923c4a3c53772140d9877f0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"3b7a7bf63bccef1e745b4f8f6507bf4c5bcf0506c6ce750e6970677fb0f50399","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"72bbfe9d9170a9135f3c05006adad3408c43107f264ad7ae326669faadef2c9f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"79926993dbe90889442ba2b21af223fbbe8febc0cc9c8b33332060f1c383fcc5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"9ba23246eb42a095691a46b9cbd632f747017108f5eaa0f29a11cd9f1796fc29","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"d0717dc22e468ddd022ff528d4aecb3fdbb5c6dab0c4a12be5be6d1d52eb794c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"d15d41907b1259276c4ebb3dec5f8bc7b80cc8a5f9d4f453ea811cdba4d45fc6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"e6aef335aed58df551668c1c4fb6e2b1c9955eae709dc370d9435a1d3a5cdffd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"f15a96cf1199ee286ffca5a7a3a92b706c70d32e3672375967fe435f387de27b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"fa90fc4c3a8dc536971855f197279bd90253d75f6a03c9cea859289d5028f03e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"0c2eda5743cefd6dcfe9ecf740d39893dbf60d284ed5aac300adeb234f564f8f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"0e70a576c756dcd421b0a13e1846e4a5a70d8e4db210c128bf73cd9a2927613f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"3e0e267bf5b593318dc8ae60ef2869ec7a0775117cd2322f89b963a63a68c41f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"420b91d3c81d83d07724defc494257d5a7fd2f05d9cb9b50a3d99e9924d26de0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"6970bcc652bfcfa5b97b4fc669aea33f82002482e6faf25e9f132a1466275393","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"994949bc78b1800725718804f81191e64a987b1448eba49bcdbdb8afecca0b76","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"f4dcf9151c86716fa85f7d2354f36e247bbe355f6542caad32346c5877430eb7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"b11c5d16e79eae44996999b7ade5b136a19160f6ec20c9f251c29794fe437393","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":17,"runs_affected":3,"failed_runs":3,"run_impact_percent":7.6923076923076925,"anchor_occurrences":17,"after_last_push_count":17,"trend_counts":[0,0,0,0,0,17,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:80]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000a2bd60\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-arm64-vm-hcp-cluster-7sdmld\" in resourcegroup=\"arm64-vm-cluster-bhd6ww\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/arm64-vm-cluster-bhd6ww/providers/Microsoft.Resources/deployments/customer-infra-arm64-vm-hcp-cluster-7sdmld/operationStatuses/08584276071679720129\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260320T094216Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '9d0d4b1f-6cd4-46ef-9360-db352e410b43', request correlation id '086357ea-a5df-43aa-9ade-32430cd392a8'.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-arm64-vm-hcp-cluster-7sdmld\\\" in resourcegroup=\\\"arm64-vm-cluster-bhd6ww\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/arm64-vm-cluster-bhd6ww/providers/Microsoft.Resources/deployments/customer-infra-arm64-vm-hcp-cluster-7sdmld/operationStatuses/08584276071679720129\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260320T094216Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '9d0d4b1f-6cd4-46ef-9360-db352e410b43', request correlation id '086357ea-a5df-43aa-9ade-32430cd392a8'.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000a2bd40\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-arm64-vm-hcp-cluster-7sdmld\\\" in resourcegroup=\\\"arm64-vm-cluster-bhd6ww\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/arm64-vm-cluster-bhd6ww/providers/Microsoft.Resources/deployments/customer-infra-arm64-vm-hcp-cluster-7sdmld/operationStatuses/08584276071679720129\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260320T094216Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '9d0d4b1f-6cd4-46ef-9360-db352e410b43', request correlation id '086357ea-a5df-43aa-9ade-32430cd392a8'.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0008036e0\u003e{\n                ErrorCode: \"DeploymentFailed\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"654f520d-e08b-4b13-9d40-7492126aea91\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Expires\": [\"-1\"],\n             ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:80]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000a2bd60\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-arm64-vm-hcp-cluster-7sdmld\" in resourcegroup=\"arm64-vm-cluster-bhd6ww\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/arm64-vm-cluster-bhd6ww/providers/Microsoft.Resources/deployments/customer-infra-arm64-vm-hcp-cluster-7sdmld/operationStatuses/08584276071679720129\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260320T094216Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '9d0d4b1f-6cd4-46ef-9360-db352e410b43', request correlation id '086357ea-a5df-43aa-9ade-32430cd392a8'.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-arm64-vm-hcp-cluster-7sdmld\\\" in resourcegroup=\\\"arm64-vm-cluster-bhd6ww\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/arm64-vm-cluster-bhd6ww/providers/Microsoft.Resources/deployments/customer-infra-arm64-vm-hcp-cluster-7sdmld/operationStatuses/08584276071679720129\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260320T094216Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '9d0d4b1f-6cd4-46ef-9360-db352e410b43', request correlation id '086357ea-a5df-43aa-9ade-32430cd392a8'.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000a2bd40\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-arm64-vm-hcp-cluster-7sdmld\\\" in resourcegroup=\\\"arm64-vm-cluster-bhd6ww\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/arm64-vm-cluster-bhd6ww/providers/Microsoft.Resources/deployments/customer-infra-arm64-vm-hcp-cluster-7sdmld/operationStatuses/08584276071679720129\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260320T094216Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '9d0d4b1f-6cd4-46ef-9360-db352e410b43', request correlation id '086357ea-a5df-43aa-9ade-32430cd392a8'.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0008036e0\u003e{\n                ErrorCode: \"DeploymentFailed\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"654f520d-e08b-4b13-9d40-7492126aea91\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Expires\": [\"-1\"],\n             ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:81]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000bc5580\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-np-labels-taints-t59jtk\" in resourcegroup=\"rg-np-labels-taints-gt77vc\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-labels-taints-gt77vc/providers/Microsoft.Resources/deployments/customer-infra-cluster-np-labels-taints-t59jtk/operationStatuses/08584276070386670482\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260320T094442Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '7da13b4c-2074-4ed2-8674-1b1ea2937fdc', request correlation id '4075b009-7643-490d-b40a-fd5cab0af989'.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-cluster-np-labels-taints-t59jtk\\\" in resourcegroup=\\\"rg-np-labels-taints-gt77vc\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-labels-taints-gt77vc/providers/Microsoft.Resources/deployments/customer-infra-cluster-np-labels-taints-t59jtk/operationStatuses/08584276070386670482\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260320T094442Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '7da13b4c-2074-4ed2-8674-1b1ea2937fdc', request correlation id '4075b009-7643-490d-b40a-fd5cab0af989'.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000bc5560\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-cluster-np-labels-taints-t59jtk\\\" in resourcegroup=\\\"rg-np-labels-taints-gt77vc\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-labels-taints-gt77vc/providers/Microsoft.Resources/deployments/customer-infra-cluster-np-labels-taints-t59jtk/operationStatuses/08584276070386670482\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260320T094442Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '7da13b4c-2074-4ed2-8674-1b1ea2937fdc', request correlation id '4075b009-7643-490d-b40a-fd5cab0af989'.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0011eaea0\u003e{\n                ErrorCode: \"DeploymentFailed\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Request-Id\": [\n                            \"af79b60e-5587-4117-9e37-523f09825369\",\n                        ],\n                        \"Date\": [\n                            \"Fri, 20 Mar 2026 09:44:49 GMT\",\n                        ],\n                        \"Content-Type\": [\n                            \"application/json; charset=utf-8\",\n                        ],\n                        \"Expires\": [\"-1\"],\n  ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:81]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000bc5580\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-np-labels-taints-t59jtk\" in resourcegroup=\"rg-np-labels-taints-gt77vc\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-labels-taints-gt77vc/providers/Microsoft.Resources/deployments/customer-infra-cluster-np-labels-taints-t59jtk/operationStatuses/08584276070386670482\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260320T094442Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '7da13b4c-2074-4ed2-8674-1b1ea2937fdc', request correlation id '4075b009-7643-490d-b40a-fd5cab0af989'.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-cluster-np-labels-taints-t59jtk\\\" in resourcegroup=\\\"rg-np-labels-taints-gt77vc\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-labels-taints-gt77vc/providers/Microsoft.Resources/deployments/customer-infra-cluster-np-labels-taints-t59jtk/operationStatuses/08584276070386670482\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260320T094442Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '7da13b4c-2074-4ed2-8674-1b1ea2937fdc', request correlation id '4075b009-7643-490d-b40a-fd5cab0af989'.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000bc5560\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-cluster-np-labels-taints-t59jtk\\\" in resourcegroup=\\\"rg-np-labels-taints-gt77vc\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-labels-taints-gt77vc/providers/Microsoft.Resources/deployments/customer-infra-cluster-np-labels-taints-t59jtk/operationStatuses/08584276070386670482\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260320T094442Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '7da13b4c-2074-4ed2-8674-1b1ea2937fdc', request correlation id '4075b009-7643-490d-b40a-fd5cab0af989'.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0011eaea0\u003e{\n                ErrorCode: \"DeploymentFailed\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Request-Id\": [\n                            \"af79b60e-5587-4117-9e37-523f09825369\",\n                        ],\n                        \"Date\": [\n                            \"Fri, 20 Mar 2026 09:44:49 GMT\",\n                        ],\n                        \"Content-Type\": [\n                            \"application/json; charset=utf-8\",\n                        ],\n                        \"Expires\": [\"-1\"],\n  ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000f82360\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-19-6fd9wv-x2gpnw\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-6fd9wv-x2gpnw/operationStatuses/08584276067733222695\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260320T094914Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'f9637488-a80a-49b4-9d85-d6a0f597d8f4', request correlation id 'c6f7a3ea-7476-48bb-9fbd-25ee8524741c'.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-19-6fd9wv-x2gpnw\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-6fd9wv-x2gpnw/operationStatuses/08584276067733222695\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260320T094914Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'f9637488-a80a-49b4-9d85-d6a0f597d8f4', request correlation id 'c6f7a3ea-7476-48bb-9fbd-25ee8524741c'.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000f82340\u003e{\n            msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-19-6fd9wv-x2gpnw\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-6fd9wv-x2gpnw/operationStatuses/08584276067733222695\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260320T094914Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'f9637488-a80a-49b4-9d85-d6a0f597d8f4', request correlation id 'c6f7a3ea-7476-48bb-9fbd-25ee8524741c'.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*fmt.wrapError | 0xc000f82320\u003e{\n                msg: \"failed waiting for deployment \\\"mi-cluster-zstream-4-19-6fd9wv-x2gpnw\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-6fd9wv-x2gpnw/operationStatuses/08584276067733222695\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:85]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000f82360\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-zstream-4-19-6fd9wv-x2gpnw\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-6fd9wv-x2gpnw/operationStatuses/08584276067733222695\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260320T094914Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'f9637488-a80a-49b4-9d85-d6a0f597d8f4', request correlation id 'c6f7a3ea-7476-48bb-9fbd-25ee8524741c'.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-19-6fd9wv-x2gpnw\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-6fd9wv-x2gpnw/operationStatuses/08584276067733222695\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260320T094914Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'f9637488-a80a-49b4-9d85-d6a0f597d8f4', request correlation id 'c6f7a3ea-7476-48bb-9fbd-25ee8524741c'.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000f82340\u003e{\n            msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-cluster-zstream-4-19-6fd9wv-x2gpnw\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-6fd9wv-x2gpnw/operationStatuses/08584276067733222695\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260320T094914Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'f9637488-a80a-49b4-9d85-d6a0f597d8f4', request correlation id 'c6f7a3ea-7476-48bb-9fbd-25ee8524741c'.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*fmt.wrapError | 0xc000f82320\u003e{\n                msg: \"failed waiting for deployment \\\"mi-cluster-zstream-4-19-6fd9wv-x2gpnw\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-zstream-4-19-6fd9wv-x2gpnw/operationStatuses/08584276067733222695\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034927649574359040","occurred_at":"2026-03-20T09:38:45Z","signature_id":"270f57b78ac0c1ee361814e4a83fa8017a43520ec1c248637d7b26332ab55e1f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034927649574359040","occurred_at":"2026-03-20T09:38:45Z","signature_id":"2f9cdebe62ff552afde2d003726143ce56e73af653d0cb26c4b58c7b1d832e40","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034927649574359040","occurred_at":"2026-03-20T09:38:45Z","signature_id":"6ac35292932e8121d3d327a5b657931cae0e1e04a1fa28af2161322539081204","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034927649574359040","occurred_at":"2026-03-20T09:38:45Z","signature_id":"86124984ab0a31ae18801d523869f691c0e71ff710a479686a9c9cd8104a4e03","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034927649574359040","occurred_at":"2026-03-20T09:38:45Z","signature_id":"bea31b99f6399e70e28a7e2ce84709292d74e0def0eceabe0fa7a6848e149191","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034927649574359040","occurred_at":"2026-03-20T09:38:45Z","signature_id":"d56e9c748212f3823af121086e352490adce276f56270d4a6836e4e1c8fd7f71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034927649574359040","occurred_at":"2026-03-20T09:38:45Z","signature_id":"f205b67bd412c4e91f3d1a644abf58519bd5346912a42e01887c0cc3d9fffc79","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034927649574359040","occurred_at":"2026-03-20T09:38:45Z","signature_id":"fa0197d185427c486cd3453c919a115d480c21e559d8e2c2ef0a0a0a0f1becca","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034909477634838528","occurred_at":"2026-03-20T08:26:40Z","signature_id":"3595c9e69aacb5bf97161bdd016c99942d44a22f77205e416114de7756621aa7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034909477634838528","occurred_at":"2026-03-20T08:26:40Z","signature_id":"6f62c377d47d3da24051e9fa65f708e151c87fe42d7d9b8ebe2797af39cac613","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034909477634838528","occurred_at":"2026-03-20T08:26:40Z","signature_id":"b38a99f6ea1f4e7eaab1b2783f6e5fff58ee19335ff36a097a20840ea2acbc0d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034909477634838528","occurred_at":"2026-03-20T08:26:40Z","signature_id":"d0a3e298fd170230fe0a59022e4870e311c4b93840bbae917daafd73c7109d2b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034884860761346048","occurred_at":"2026-03-20T06:48:51Z","signature_id":"2f998672fa842e53d96ec220358aa27944e385cefa102210558d4e6feaf57107","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034884860761346048","occurred_at":"2026-03-20T06:48:51Z","signature_id":"327bca80b464a7d9aaf4eef23a0824096ee3e3d74ce347798990e8f5111e30c4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034884860761346048","occurred_at":"2026-03-20T06:48:51Z","signature_id":"4eb3e4d73e5c65c9171a8081718a49c6a59dad812d1b93e7937c743e3e80db20","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034884860761346048","occurred_at":"2026-03-20T06:48:51Z","signature_id":"700ed2fa3d1d5e4da61ba25287538176794f67d88b7d0a73cb0b62948e614312","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034884860761346048","occurred_at":"2026-03-20T06:48:51Z","signature_id":"bcdfd7f494c086da951bf8677614af92afab7e3f115b7461e78a41c681a76b14","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-m559f4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-m559f4, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":16,"runs_affected":2,"failed_runs":2,"run_impact_percent":5.128205128205129,"anchor_occurrences":16,"after_last_push_count":16,"trend_counts":[0,9,0,7,0,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-02"],"prior_runs_affected":8,"prior_last_seen_at":"2026-03-07T02:00:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:96]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e6cb10\u003e: \n    failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-vm79bx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vm79bx\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0011648c0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000e6cae0\u003e{\n                msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vm79bx\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0011648c0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:96]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e6cb10\u003e: \n    failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-vm79bx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vm79bx\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0011648c0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000e6cae0\u003e{\n                msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vm79bx\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0011648c0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00101e2d0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-nxd528\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-nxd528\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001374460\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00101e2a0\u003e{\n                msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-nxd528\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001374460\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00101e2d0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-nxd528\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-nxd528\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001374460\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00101e2a0\u003e{\n                msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-nxd528\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001374460\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00142ccf0\u003e: \n    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-sjngpl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-sjngpl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00158dc10\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00142ccc0\u003e{\n                msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-sjngpl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc00158dc10\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00142ccf0\u003e: \n    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-sjngpl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-sjngpl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00158dc10\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00142ccc0\u003e{\n                msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-sjngpl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc00158dc10\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"0adb0e14ff90a50fd727e2352980ce875ced0b7130a59c2b7ed94a1d2accd443","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"20b718cbd2e3ab1df68073123c329c2df9c689fd937380268f7e88c5d24e952c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"29f0eae22dce4003fd469e952b9869eb3891a2ef3f974a873ee4dd1a59a06db0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"aea60a8b0ab44c615808be854a956d9a8caed07cbc9500e67544d4ee655aa8d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"b3d5ff4c6489e7b7eebdfc119d86db673a1197b69b52d19f8423004d3f858303","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"ba32603bd52a75ed7762808e5de2a5d3ad0e6b5a105d17eda82735596cad91f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"ebdd99780dc0ca89741307789c856554dc35c473e657c97cf89059a55b98274d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"08e328dc1dfd1fb161e5ee5fbeb36a333526cb48a4a6251392be2ebade86d839","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"1eb3805a5fe71ec50edd4d69c587ec8350ab24f91e053ebfd61eded6cdf1d4ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"3b352a408296d3a0251d7a6d14862960baca6761c1863c44d92551e765866899","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"488befe4edc4450b2b46b4d02c1793f8c3dd4e75a7c8db29f6ea6f12b2d9673b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"5aeff6efa147f0a4f4455f0d12d6e118c4fd736fa1d304c8919d5fa6389a770e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"a3917a20697f535d483e2f50596dc25ed08eb3cecf016c87a084b621b064671c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"d96df068ad434e9a0e2966e4bb5122791524d6797a32adea9a72c57456b99c47","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"e84f3d91e58a4ddeb3430bb8931aee0419956b91f0597cc4dde0c7443f9de91f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"f84db3c137cad46a28c3e54119e47a5ebf7698ff6bc2244e20e068bd0436bdd4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"404b4766e52f5ee2ac5a708649d75d362172856f013ef26f84427d67f1ddce29","failure_pattern":"or adjust the parameters in Gomega's 'format' package.","search_query":"or adjust the parameters in Gomega's 'format' package.","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":15.384615384615385,"anchor_occurrences":6,"after_last_push_count":7,"trend_counts":[0,0,0,0,6,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-09"],"prior_runs_affected":7,"prior_last_seen_at":"2026-03-11T00:41:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","occurrences":6}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:97]: Expected\n    \u003c*exported.ResponseError | 0xc0014e61e0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: ProviderError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ProviderError\",\n        \"message\": \"Resource provider 'Microsoft.RedHatOpenShift' failed to return collection response for type 'HCPOpenShiftClusters'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        ErrorCode: \"ProviderError\",\n        StatusCode: 502,\n        RawResponse: {\n            Status: \"502 Bad Gateway\",\n            StatusCode: 502,\n            Proto: \"HTTP/2.0\",\n            ProtoMajor: 2,\n            ProtoMinor: 0,\n            Header: {\n                \"X-Ms-Routing-Request-Id\": [\n                    \"CANADACENTRAL:20260319T155939Z:81a9fbb3-a426-4d48-a5cc-b6e263995e30\",\n                ],\n                \"X-Msedge-Ref\": [\n                    \"Ref A: 59C38C75B1404B55A084E8EB38DC0136 Ref B: CHI261140501029 Ref C: 2026-03-19T15:58:38Z\",\n                ],\n                \"Date\": [\n                    \"Thu, 19 Mar 2026 15:59:39 GMT\",\n                ],\n                \"Cache-Control\": [\"no-cache\"],\n                \"Content-Length\": [\"162\"],\n                \"Content-Type\": [\n                    \"application/json; charset=utf-8\",\n                ],\n                \"Strict-Transport-Security\": [\n                    \"max-age=31536000; includeSubDomains\",\n                ],\n                \"X-Content-Type-Options\": [\"nosniff\"],\n                \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                \"Pragma\": [\"no-cache\"],\n                \"Expires\": [\"-1\"],\n                \"X-Ms-Failure-Cause\": [\"service\"],\n                \"X-Ms-Request-Id\": [\n                    \"81a9fbb3-a426-4d48-a5cc-b6e263995e30\",\n                ],\n                \"X-Ms-Correlation-Request-Id\": [\n                    \"81a9fbb3-a426-4d48-a5cc-b6e263995e30\",\n                ],\n            },\n            Body: \u003c*exported.nopClosingBytesReader | 0xc000c14000\u003e{\n                s: \"{\\\"error\\\":{\\\"code\\\":\\\"ProviderError\\\",\\\"message\\\":\\\"Resource provider 'Microsoft.RedHatOpenShift' failed to return collection response for type 'HCPOpenShiftClusters'.\\\"}}\",\n                i: 0,\n            },\n            ContentLength: 162,\n            TransferEncoding: nil,\n            Close: false,\n            Uncompressed: false,\n            Trailer: nil,\n            Request: {\n                Method: \"GET\",\n                URL: {\n                    Scheme: \"https\",\n                    Opaque: \"\",\n                    User: nil,\n                    Host: \"management.azure.com\",\n                    Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters\",\n                    RawPath: \"\",\n                    OmitHost: false,\n                    ForceQuery: false,\n                    RawQuery: \"%24skiptoken=1Y%2FBaoQwGITf5T9vjLrZbhVKkWWhh0rLunvzEs2vBtckJNGWiu%2Fe0EPpK%2FQ2MwzMfCso%2FPSvUo0O8hXORXW9Ven5VrxDDoP3xuWUTlzxHidUPuJfs8Wo1RN1c%2BNaK42XWjnK4r3IBGYk2acNYdmRkYzxAzk0x8dGdDzJEqTG6kUKtI6WsrXa6c5HFxQv3L8ZVNUgO0%2BH1vya0312PtSfuZFkCSJMPaVxykj8QJKYGIuLxA%2FYdsDtVPS9xZ57FFc9ogr%2Fi0sJux%2FAktsRbYjW%2Bi9jDXn9Xyhr2GD7Bg%3D%3D\u0026api-version=2024-06-10-preview\",\n                    Fragment: \"\",\n                    RawFragment: \"\",\n                },\n                Proto: \"HTTP/1.1\",\n                ProtoMajor: 1,\n                ProtoMinor: 1,\n                Header: {\n                    \"Authorization\": [\"redacted\"],\n                    \"User-Agent\": [\n                        \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7; linux)\",\n                    ],\n                },\n                Body: nil,\n                GetBody: nil,\n                ContentLength: 0,\n                TransferEncoding: nil,\n                Close: false,\n                Host: \"management.azure.com\",\n                Form: nil,\n                PostForm: nil,\n                MultipartForm: nil,\n                Trailer: nil,\n                RemoteAddr: \"\",\n                RequestURI: \"\",\n                TLS: nil,\n                Cancel: nil,\n                Response: nil,\n                Pattern: \"\",\n                ctx: \u003csuppressed context\u003e,\n                pat: nil,\n                matches: nil,\n                otherValues: nil,\n            },\n            TLS: {\n                Version: 772,\n                HandshakeComplete: true,\n             ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\nto be nil\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:97]: Expected\n    \u003c*exported.ResponseError | 0xc0014e61e0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: ProviderError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ProviderError\",\n        \"message\": \"Resource provider 'Microsoft.RedHatOpenShift' failed to return collection response for type 'HCPOpenShiftClusters'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        ErrorCode: \"ProviderError\",\n        StatusCode: 502,\n        RawResponse: {\n            Status: \"502 Bad Gateway\",\n            StatusCode: 502,\n            Proto: \"HTTP/2.0\",\n            ProtoMajor: 2,\n            ProtoMinor: 0,\n            Header: {\n                \"X-Ms-Routing-Request-Id\": [\n                    \"CANADACENTRAL:20260319T155939Z:81a9fbb3-a426-4d48-a5cc-b6e263995e30\",\n                ],\n                \"X-Msedge-Ref\": [\n                    \"Ref A: 59C38C75B1404B55A084E8EB38DC0136 Ref B: CHI261140501029 Ref C: 2026-03-19T15:58:38Z\",\n                ],\n                \"Date\": [\n                    \"Thu, 19 Mar 2026 15:59:39 GMT\",\n                ],\n                \"Cache-Control\": [\"no-cache\"],\n                \"Content-Length\": [\"162\"],\n                \"Content-Type\": [\n                    \"application/json; charset=utf-8\",\n                ],\n                \"Strict-Transport-Security\": [\n                    \"max-age=31536000; includeSubDomains\",\n                ],\n                \"X-Content-Type-Options\": [\"nosniff\"],\n                \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                \"Pragma\": [\"no-cache\"],\n                \"Expires\": [\"-1\"],\n                \"X-Ms-Failure-Cause\": [\"service\"],\n                \"X-Ms-Request-Id\": [\n                    \"81a9fbb3-a426-4d48-a5cc-b6e263995e30\",\n                ],\n                \"X-Ms-Correlation-Request-Id\": [\n                    \"81a9fbb3-a426-4d48-a5cc-b6e263995e30\",\n                ],\n            },\n            Body: \u003c*exported.nopClosingBytesReader | 0xc000c14000\u003e{\n                s: \"{\\\"error\\\":{\\\"code\\\":\\\"ProviderError\\\",\\\"message\\\":\\\"Resource provider 'Microsoft.RedHatOpenShift' failed to return collection response for type 'HCPOpenShiftClusters'.\\\"}}\",\n                i: 0,\n            },\n            ContentLength: 162,\n            TransferEncoding: nil,\n            Close: false,\n            Uncompressed: false,\n            Trailer: nil,\n            Request: {\n                Method: \"GET\",\n                URL: {\n                    Scheme: \"https\",\n                    Opaque: \"\",\n                    User: nil,\n                    Host: \"management.azure.com\",\n                    Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters\",\n                    RawPath: \"\",\n                    OmitHost: false,\n                    ForceQuery: false,\n                    RawQuery: \"%24skiptoken=1Y%2FBaoQwGITf5T9vjLrZbhVKkWWhh0rLunvzEs2vBtckJNGWiu%2Fe0EPpK%2FQ2MwzMfCso%2FPSvUo0O8hXORXW9Ven5VrxDDoP3xuWUTlzxHidUPuJfs8Wo1RN1c%2BNaK42XWjnK4r3IBGYk2acNYdmRkYzxAzk0x8dGdDzJEqTG6kUKtI6WsrXa6c5HFxQv3L8ZVNUgO0%2BH1vya0312PtSfuZFkCSJMPaVxykj8QJKYGIuLxA%2FYdsDtVPS9xZ57FFc9ogr%2Fi0sJux%2FAktsRbYjW%2Bi9jDXn9Xyhr2GD7Bg%3D%3D\u0026api-version=2024-06-10-preview\",\n                    Fragment: \"\",\n                    RawFragment: \"\",\n                },\n                Proto: \"HTTP/1.1\",\n                ProtoMajor: 1,\n                ProtoMinor: 1,\n                Header: {\n                    \"Authorization\": [\"redacted\"],\n                    \"User-Agent\": [\n                        \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7; linux)\",\n                    ],\n                },\n                Body: nil,\n                GetBody: nil,\n                ContentLength: 0,\n                TransferEncoding: nil,\n                Close: false,\n                Host: \"management.azure.com\",\n                Form: nil,\n                PostForm: nil,\n                MultipartForm: nil,\n                Trailer: nil,\n                RemoteAddr: \"\",\n                RequestURI: \"\",\n                TLS: nil,\n                Cancel: nil,\n                Response: nil,\n                Pattern: \"\",\n                ctx: \u003csuppressed context\u003e,\n                pat: nil,\n                matches: nil,\n                otherValues: nil,\n            },\n            TLS: {\n                Version: 772,\n                HandshakeComplete: true,\n             ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\nto be nil","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:97]: Expected\n    \u003c*exported.ResponseError | 0xc000c953e0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: ProviderError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ProviderError\",\n        \"message\": \"Resource provider 'Microsoft.RedHatOpenShift' failed to return collection response for type 'HCPOpenShiftClusters'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        ErrorCode: \"ProviderError\",\n        StatusCode: 502,\n        RawResponse: {\n            Status: \"502 Bad Gateway\",\n            StatusCode: 502,\n            Proto: \"HTTP/2.0\",\n            ProtoMajor: 2,\n            ProtoMinor: 0,\n            Header: {\n                \"Content-Type\": [\n                    \"application/json; charset=utf-8\",\n                ],\n                \"Expires\": [\"-1\"],\n                \"Strict-Transport-Security\": [\n                    \"max-age=31536000; includeSubDomains\",\n                ],\n                \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                \"Date\": [\n                    \"Thu, 19 Mar 2026 10:24:04 GMT\",\n                ],\n                \"Cache-Control\": [\"no-cache\"],\n                \"Content-Length\": [\"162\"],\n                \"X-Ms-Failure-Cause\": [\"service\"],\n                \"X-Ms-Request-Id\": [\n                    \"a16a20c5-d011-43da-ae02-96abf758f533\",\n                ],\n                \"X-Ms-Correlation-Request-Id\": [\n                    \"a16a20c5-d011-43da-ae02-96abf758f533\",\n                ],\n                \"X-Ms-Routing-Request-Id\": [\n                    \"CANADACENTRAL:20260319T102404Z:a16a20c5-d011-43da-ae02-96abf758f533\",\n                ],\n                \"X-Content-Type-Options\": [\"nosniff\"],\n                \"X-Msedge-Ref\": [\n                    \"Ref A: FD20AB1260094084B33CC10F073A5B77 Ref B: CHI261140514052 Ref C: 2026-03-19T10:23:58Z\",\n                ],\n                \"Pragma\": [\"no-cache\"],\n            },\n            Body: \u003c*exported.nopClosingBytesReader | 0xc000f9a080\u003e{\n                s: \"{\\\"error\\\":{\\\"code\\\":\\\"ProviderError\\\",\\\"message\\\":\\\"Resource provider 'Microsoft.RedHatOpenShift' failed to return collection response for type 'HCPOpenShiftClusters'.\\\"}}\",\n                i: 0,\n            },\n            ContentLength: 162,\n            TransferEncoding: nil,\n            Close: false,\n            Uncompressed: false,\n            Trailer: nil,\n            Request: {\n                Method: \"GET\",\n                URL: {\n                    Scheme: \"https\",\n                    Opaque: \"\",\n                    User: nil,\n                    Host: \"management.azure.com\",\n                    Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters\",\n                    RawPath: \"\",\n                    OmitHost: false,\n                    ForceQuery: false,\n                    RawQuery: \"%24skiptoken=1Y%2FBaoQwGITf5T9vjLrZbhVKkWWhh0rLunvzEs2vBtckJNGWiu%2Fe0EPpK%2FQ2MwzMfCso%2FPSvUo0O8hXORXW9Ven5VrxDDoP3xuWUTlzxHidUPuJfs8Wo1RN1c%2BNaK42XWjnK4r3IBGYk2acNYdmRkYzxAzk0x8dGdDzJEqTG6kUKtI6WsrXa6c5HFxQv3L8ZVNUgO0%2BH1vya0312PtSfuZFkCSJMPaVxykj8QJKYGIuLxA%2FYdsDtVPS9xZ57FFc9ogr%2Fi0sJux%2FAktsRbYjW%2Bi9jDXn9Xyhr2GD7Bg%3D%3D\u0026api-version=2024-06-10-preview\",\n                    Fragment: \"\",\n                    RawFragment: \"\",\n                },\n                Proto: \"HTTP/1.1\",\n                ProtoMajor: 1,\n                ProtoMinor: 1,\n                Header: {\n                    \"User-Agent\": [\n                        \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7; linux)\",\n                    ],\n                    \"Authorization\": [\"redacted\"],\n                },\n                Body: nil,\n                GetBody: nil,\n                ContentLength: 0,\n                TransferEncoding: nil,\n                Close: false,\n                Host: \"management.azure.com\",\n                Form: nil,\n                PostForm: nil,\n                MultipartForm: nil,\n                Trailer: nil,\n                RemoteAddr: \"\",\n                RequestURI: \"\",\n                TLS: nil,\n                Cancel: nil,\n                Response: nil,\n                Pattern: \"\",\n                ctx: \u003csuppressed context\u003e,\n                pat: nil,\n                matches: nil,\n                otherValues: nil,\n            },\n            TLS: {\n                Version: 772,\n                HandshakeComplete: true,\n             ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\nto be nil\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:97]: Expected\n    \u003c*exported.ResponseError | 0xc000c953e0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: ProviderError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ProviderError\",\n        \"message\": \"Resource provider 'Microsoft.RedHatOpenShift' failed to return collection response for type 'HCPOpenShiftClusters'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        ErrorCode: \"ProviderError\",\n        StatusCode: 502,\n        RawResponse: {\n            Status: \"502 Bad Gateway\",\n            StatusCode: 502,\n            Proto: \"HTTP/2.0\",\n            ProtoMajor: 2,\n            ProtoMinor: 0,\n            Header: {\n                \"Content-Type\": [\n                    \"application/json; charset=utf-8\",\n                ],\n                \"Expires\": [\"-1\"],\n                \"Strict-Transport-Security\": [\n                    \"max-age=31536000; includeSubDomains\",\n                ],\n                \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                \"Date\": [\n                    \"Thu, 19 Mar 2026 10:24:04 GMT\",\n                ],\n                \"Cache-Control\": [\"no-cache\"],\n                \"Content-Length\": [\"162\"],\n                \"X-Ms-Failure-Cause\": [\"service\"],\n                \"X-Ms-Request-Id\": [\n                    \"a16a20c5-d011-43da-ae02-96abf758f533\",\n                ],\n                \"X-Ms-Correlation-Request-Id\": [\n                    \"a16a20c5-d011-43da-ae02-96abf758f533\",\n                ],\n                \"X-Ms-Routing-Request-Id\": [\n                    \"CANADACENTRAL:20260319T102404Z:a16a20c5-d011-43da-ae02-96abf758f533\",\n                ],\n                \"X-Content-Type-Options\": [\"nosniff\"],\n                \"X-Msedge-Ref\": [\n                    \"Ref A: FD20AB1260094084B33CC10F073A5B77 Ref B: CHI261140514052 Ref C: 2026-03-19T10:23:58Z\",\n                ],\n                \"Pragma\": [\"no-cache\"],\n            },\n            Body: \u003c*exported.nopClosingBytesReader | 0xc000f9a080\u003e{\n                s: \"{\\\"error\\\":{\\\"code\\\":\\\"ProviderError\\\",\\\"message\\\":\\\"Resource provider 'Microsoft.RedHatOpenShift' failed to return collection response for type 'HCPOpenShiftClusters'.\\\"}}\",\n                i: 0,\n            },\n            ContentLength: 162,\n            TransferEncoding: nil,\n            Close: false,\n            Uncompressed: false,\n            Trailer: nil,\n            Request: {\n                Method: \"GET\",\n                URL: {\n                    Scheme: \"https\",\n                    Opaque: \"\",\n                    User: nil,\n                    Host: \"management.azure.com\",\n                    Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters\",\n                    RawPath: \"\",\n                    OmitHost: false,\n                    ForceQuery: false,\n                    RawQuery: \"%24skiptoken=1Y%2FBaoQwGITf5T9vjLrZbhVKkWWhh0rLunvzEs2vBtckJNGWiu%2Fe0EPpK%2FQ2MwzMfCso%2FPSvUo0O8hXORXW9Ven5VrxDDoP3xuWUTlzxHidUPuJfs8Wo1RN1c%2BNaK42XWjnK4r3IBGYk2acNYdmRkYzxAzk0x8dGdDzJEqTG6kUKtI6WsrXa6c5HFxQv3L8ZVNUgO0%2BH1vya0312PtSfuZFkCSJMPaVxykj8QJKYGIuLxA%2FYdsDtVPS9xZ57FFc9ogr%2Fi0sJux%2FAktsRbYjW%2Bi9jDXn9Xyhr2GD7Bg%3D%3D\u0026api-version=2024-06-10-preview\",\n                    Fragment: \"\",\n                    RawFragment: \"\",\n                },\n                Proto: \"HTTP/1.1\",\n                ProtoMajor: 1,\n                ProtoMinor: 1,\n                Header: {\n                    \"User-Agent\": [\n                        \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7; linux)\",\n                    ],\n                    \"Authorization\": [\"redacted\"],\n                },\n                Body: nil,\n                GetBody: nil,\n                ContentLength: 0,\n                TransferEncoding: nil,\n                Close: false,\n                Host: \"management.azure.com\",\n                Form: nil,\n                PostForm: nil,\n                MultipartForm: nil,\n                Trailer: nil,\n                RemoteAddr: \"\",\n                RequestURI: \"\",\n                TLS: nil,\n                Cancel: nil,\n                Response: nil,\n                Pattern: \"\",\n                ctx: \u003csuppressed context\u003e,\n                pat: nil,\n                matches: nil,\n                otherValues: nil,\n            },\n            TLS: {\n                Version: 772,\n                HandshakeComplete: true,\n             ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\nto be nil","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:97]: Expected\n    \u003c*exported.ResponseError | 0xc000f8ea50\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: ProviderError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ProviderError\",\n        \"message\": \"Resource provider 'Microsoft.RedHatOpenShift' failed to return collection response for type 'HCPOpenShiftClusters'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        ErrorCode: \"ProviderError\",\n        StatusCode: 502,\n        RawResponse: {\n            Status: \"502 Bad Gateway\",\n            StatusCode: 502,\n            Proto: \"HTTP/2.0\",\n            ProtoMajor: 2,\n            ProtoMinor: 0,\n            Header: {\n                \"Pragma\": [\"no-cache\"],\n                \"Expires\": [\"-1\"],\n                \"X-Ms-Request-Id\": [\n                    \"d6d059ca-f5bf-439e-bbfd-46f05c53e078\",\n                ],\n                \"X-Ms-Correlation-Request-Id\": [\n                    \"d6d059ca-f5bf-439e-bbfd-46f05c53e078\",\n                ],\n                \"X-Content-Type-Options\": [\"nosniff\"],\n                \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                \"X-Msedge-Ref\": [\n                    \"Ref A: 43F7E9279933424097790033A2000E86 Ref B: CHI261140509034 Ref C: 2026-03-19T10:24:59Z\",\n                ],\n                \"Cache-Control\": [\"no-cache\"],\n                \"Content-Length\": [\"162\"],\n                \"Content-Type\": [\n                    \"application/json; charset=utf-8\",\n                ],\n                \"X-Ms-Failure-Cause\": [\"service\"],\n                \"X-Ms-Routing-Request-Id\": [\n                    \"CANADACENTRAL:20260319T102559Z:d6d059ca-f5bf-439e-bbfd-46f05c53e078\",\n                ],\n                \"Strict-Transport-Security\": [\n                    \"max-age=31536000; includeSubDomains\",\n                ],\n                \"Date\": [\n                    \"Thu, 19 Mar 2026 10:25:59 GMT\",\n                ],\n            },\n            Body: \u003c*exported.nopClosingBytesReader | 0xc0008f6380\u003e{\n                s: \"{\\\"error\\\":{\\\"code\\\":\\\"ProviderError\\\",\\\"message\\\":\\\"Resource provider 'Microsoft.RedHatOpenShift' failed to return collection response for type 'HCPOpenShiftClusters'.\\\"}}\",\n                i: 0,\n            },\n            ContentLength: 162,\n            TransferEncoding: nil,\n            Close: false,\n            Uncompressed: false,\n            Trailer: nil,\n            Request: {\n                Method: \"GET\",\n                URL: {\n                    Scheme: \"https\",\n                    Opaque: \"\",\n                    User: nil,\n                    Host: \"management.azure.com\",\n                    Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters\",\n                    RawPath: \"\",\n                    OmitHost: false,\n                    ForceQuery: false,\n                    RawQuery: \"%24skiptoken=1Y%2FBaoQwGITf5T9vjLrZbhVKkWWhh0rLunvzEs2vBtckJNGWiu%2Fe0EPpK%2FQ2MwzMfCso%2FPSvUo0O8hXORXW9Ven5VrxDDoP3xuWUTlzxHidUPuJfs8Wo1RN1c%2BNaK42XWjnK4r3IBGYk2acNYdmRkYzxAzk0x8dGdDzJEqTG6kUKtI6WsrXa6c5HFxQv3L8ZVNUgO0%2BH1vya0312PtSfuZFkCSJMPaVxykj8QJKYGIuLxA%2FYdsDtVPS9xZ57FFc9ogr%2Fi0sJux%2FAktsRbYjW%2Bi9jDXn9Xyhr2GD7Bg%3D%3D\u0026api-version=2024-06-10-preview\",\n                    Fragment: \"\",\n                    RawFragment: \"\",\n                },\n                Proto: \"HTTP/1.1\",\n                ProtoMajor: 1,\n                ProtoMinor: 1,\n                Header: {\n                    \"User-Agent\": [\n                        \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7; linux)\",\n                    ],\n                    \"Authorization\": [\"redacted\"],\n                },\n                Body: nil,\n                GetBody: nil,\n                ContentLength: 0,\n                TransferEncoding: nil,\n                Close: false,\n                Host: \"management.azure.com\",\n                Form: nil,\n                PostForm: nil,\n                MultipartForm: nil,\n                Trailer: nil,\n                RemoteAddr: \"\",\n                RequestURI: \"\",\n                TLS: nil,\n                Cancel: nil,\n                Response: nil,\n                Pattern: \"\",\n                ctx: \u003csuppressed context\u003e,\n                pat: nil,\n                matches: nil,\n                otherValues: nil,\n            },\n            TLS: {\n                Version: 772,\n                HandshakeComplete: true,\n             ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\nto be nil\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:97]: Expected\n    \u003c*exported.ResponseError | 0xc000f8ea50\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: ProviderError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ProviderError\",\n        \"message\": \"Resource provider 'Microsoft.RedHatOpenShift' failed to return collection response for type 'HCPOpenShiftClusters'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        ErrorCode: \"ProviderError\",\n        StatusCode: 502,\n        RawResponse: {\n            Status: \"502 Bad Gateway\",\n            StatusCode: 502,\n            Proto: \"HTTP/2.0\",\n            ProtoMajor: 2,\n            ProtoMinor: 0,\n            Header: {\n                \"Pragma\": [\"no-cache\"],\n                \"Expires\": [\"-1\"],\n                \"X-Ms-Request-Id\": [\n                    \"d6d059ca-f5bf-439e-bbfd-46f05c53e078\",\n                ],\n                \"X-Ms-Correlation-Request-Id\": [\n                    \"d6d059ca-f5bf-439e-bbfd-46f05c53e078\",\n                ],\n                \"X-Content-Type-Options\": [\"nosniff\"],\n                \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                \"X-Msedge-Ref\": [\n                    \"Ref A: 43F7E9279933424097790033A2000E86 Ref B: CHI261140509034 Ref C: 2026-03-19T10:24:59Z\",\n                ],\n                \"Cache-Control\": [\"no-cache\"],\n                \"Content-Length\": [\"162\"],\n                \"Content-Type\": [\n                    \"application/json; charset=utf-8\",\n                ],\n                \"X-Ms-Failure-Cause\": [\"service\"],\n                \"X-Ms-Routing-Request-Id\": [\n                    \"CANADACENTRAL:20260319T102559Z:d6d059ca-f5bf-439e-bbfd-46f05c53e078\",\n                ],\n                \"Strict-Transport-Security\": [\n                    \"max-age=31536000; includeSubDomains\",\n                ],\n                \"Date\": [\n                    \"Thu, 19 Mar 2026 10:25:59 GMT\",\n                ],\n            },\n            Body: \u003c*exported.nopClosingBytesReader | 0xc0008f6380\u003e{\n                s: \"{\\\"error\\\":{\\\"code\\\":\\\"ProviderError\\\",\\\"message\\\":\\\"Resource provider 'Microsoft.RedHatOpenShift' failed to return collection response for type 'HCPOpenShiftClusters'.\\\"}}\",\n                i: 0,\n            },\n            ContentLength: 162,\n            TransferEncoding: nil,\n            Close: false,\n            Uncompressed: false,\n            Trailer: nil,\n            Request: {\n                Method: \"GET\",\n                URL: {\n                    Scheme: \"https\",\n                    Opaque: \"\",\n                    User: nil,\n                    Host: \"management.azure.com\",\n                    Path: \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters\",\n                    RawPath: \"\",\n                    OmitHost: false,\n                    ForceQuery: false,\n                    RawQuery: \"%24skiptoken=1Y%2FBaoQwGITf5T9vjLrZbhVKkWWhh0rLunvzEs2vBtckJNGWiu%2Fe0EPpK%2FQ2MwzMfCso%2FPSvUo0O8hXORXW9Ven5VrxDDoP3xuWUTlzxHidUPuJfs8Wo1RN1c%2BNaK42XWjnK4r3IBGYk2acNYdmRkYzxAzk0x8dGdDzJEqTG6kUKtI6WsrXa6c5HFxQv3L8ZVNUgO0%2BH1vya0312PtSfuZFkCSJMPaVxykj8QJKYGIuLxA%2FYdsDtVPS9xZ57FFc9ogr%2Fi0sJux%2FAktsRbYjW%2Bi9jDXn9Xyhr2GD7Bg%3D%3D\u0026api-version=2024-06-10-preview\",\n                    Fragment: \"\",\n                    RawFragment: \"\",\n                },\n                Proto: \"HTTP/1.1\",\n                ProtoMajor: 1,\n                ProtoMinor: 1,\n                Header: {\n                    \"User-Agent\": [\n                        \"azsdk-go-armredhatopenshifthcp/v0.1.0 (go1.25.7; linux)\",\n                    ],\n                    \"Authorization\": [\"redacted\"],\n                },\n                Body: nil,\n                GetBody: nil,\n                ContentLength: 0,\n                TransferEncoding: nil,\n                Close: false,\n                Host: \"management.azure.com\",\n                Form: nil,\n                PostForm: nil,\n                MultipartForm: nil,\n                Trailer: nil,\n                RemoteAddr: \"\",\n                RequestURI: \"\",\n                TLS: nil,\n                Cancel: nil,\n                Response: nil,\n                Pattern: \"\",\n                ctx: \u003csuppressed context\u003e,\n                pat: nil,\n                matches: nil,\n                otherValues: nil,\n            },\n            TLS: {\n                Version: 772,\n                HandshakeComplete: true,\n             ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\nto be nil"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034658547400183808","occurred_at":"2026-03-19T15:49:34Z","signature_id":"c5279d1745d67c1c1917e76bc790519947e843a5f592243e919fd3ca24d59ea2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034574621751119872","occurred_at":"2026-03-19T10:16:04Z","signature_id":"a0f5d482f7f4704ba5b2f6de52729495f819e315afd3c2195a3b074583e3ac58","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034572752144306176","occurred_at":"2026-03-19T10:08:39Z","signature_id":"3982d9998edb88b9f5e16a7a3b936cd31d16b8662668e45cea11898f29056746","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034544437320749056","occurred_at":"2026-03-19T08:16:08Z","signature_id":"e40957a677699dc4f93e0b61ef5c9aaf57f3a06b784e881d6994ae71ccba5bfe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034516853589544960","occurred_at":"2026-03-19T06:26:31Z","signature_id":"1b9170634bf7412b07467b53aced52049e029921d33bd845cf340541915483c4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034511630959644672","occurred_at":"2026-03-19T06:05:46Z","signature_id":"ec36a79265c86af59d099c2bd80d6d670d323dae8862ee92566d8227eaa51f78","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-dpcrt8, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-dpcrt8\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":7.6923076923076925,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,4,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":2,"prior_window_starts":["2026-02-23","2026-03-02"],"prior_runs_affected":11,"prior_last_seen_at":"2026-03-05T02:00:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:145]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009b8840\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-jmwp7b, error: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-jmwp7b\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-jmwp7b, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-jmwp7b\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0005a92b0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-jmwp7b\",\n            },\n            \u003c*fmt.wrapError | 0xc001396680\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-jmwp7b\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:145]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009b8840\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-jmwp7b, error: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-jmwp7b\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-jmwp7b, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-jmwp7b\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0005a92b0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-jmwp7b\",\n            },\n            \u003c*fmt.wrapError | 0xc001396680\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-jmwp7b\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:123]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013fc0c0\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group tls-endpoint-cluster-k8q47m, error: failed waiting for nodepool=\"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-k8q47m\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group tls-endpoint-cluster-k8q47m, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-k8q47m\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00150edc0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group tls-endpoint-cluster-k8q47m\",\n            },\n            \u003c*fmt.wrapError | 0xc000c84460\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-k8q47m\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:123]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013fc0c0\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group tls-endpoint-cluster-k8q47m, error: failed waiting for nodepool=\"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-k8q47m\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group tls-endpoint-cluster-k8q47m, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-k8q47m\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00150edc0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group tls-endpoint-cluster-k8q47m\",\n            },\n            \u003c*fmt.wrapError | 0xc000c84460\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-k8q47m\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:145]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00092ae70\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-t7lnhv, error: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-t7lnhv\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-t7lnhv, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-t7lnhv\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001288490\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-t7lnhv\",\n            },\n            \u003c*fmt.wrapError | 0xc000960c40\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-t7lnhv\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:145]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00092ae70\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-t7lnhv, error: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-t7lnhv\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-t7lnhv, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-t7lnhv\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001288490\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-t7lnhv\",\n            },\n            \u003c*fmt.wrapError | 0xc000960c40\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-t7lnhv\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034516853589544960","occurred_at":"2026-03-19T06:26:31Z","signature_id":"4569757a3ec8e313d27096ec97e16080c5ff0fc723c2c20a707a9d5c24b051c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034491906846625792","occurred_at":"2026-03-19T04:47:24Z","signature_id":"52fbaeec78339aa05f37b9f2ab0f926293feb69ab0c0d9eb2be096a734ee3609","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034491906846625792","occurred_at":"2026-03-19T04:47:24Z","signature_id":"ce4216f6b07fa99bffaf36daca2b8b1cc631085f4d5d55f076706fc6f1f54240","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034462034761879552","occurred_at":"2026-03-19T02:48:42Z","signature_id":"a1678d33ae84922537b0e259358251d14eaf63ad4cb06a36060b401ac6e19742","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":5.128205128205129,"anchor_occurrences":2,"after_last_push_count":5,"trend_counts":[0,1,0,1,0,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-02"],"prior_runs_affected":5,"prior_last_seen_at":"2026-03-05T07:27:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"b8285d1fe5e0e06b0118077c74385230269c41bde5a189b9513acacc0d1a338e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"b8285d1fe5e0e06b0118077c74385230269c41bde5a189b9513acacc0d1a338e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":5.128205128205129,"anchor_occurrences":2,"after_last_push_count":18,"trend_counts":[0,0,0,1,1,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":2,"prior_window_starts":["2026-02-23","2026-03-02"],"prior_runs_affected":12,"prior_last_seen_at":"2026-03-05T05:33:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2}],"full_error_samples":["284\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func15.2({0x7fe3c01a6a60, 0xc00085a750})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n                | \n                | By(\"waiting for first cluster to complete creation\")\n                \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n                | \tFrequency: framework.StandardPollInterval,\n                | })\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x827b940?, 0xc00085a750?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 99\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/19/26 04:21:45.455\n  \"ts\"=\"2026-03-19 04:21:45.455935\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-19 04:21:45.456006\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-qpcr94\" \"resourceGroup\"=\"customer-rg-ggrxmh\"\n  \"ts\"=\"2026-03-19 04:21:45.775140\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-qpcr94\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-19 04:21:48.084018\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-lzd56b\" \"resourceGroup\"=\"customer-rg-ggrxmh\"\n  \"ts\"=\"2026-03-19 04:21:48.957488\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-lzd56b\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-19 04:21:51.648746\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-19 04:21:51.648859\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-19 04:21:51.648920\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-19 04:21:51.650029\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-9\"\n  \"ts\"=\"2026-03-19 04:21:51.650042\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-56\"\n  \"ts\"=\"2026-03-19 04:21:51.650081\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-149\"\n  \"ts\"=\"2026-03-19 04:21:51.650073\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-ggrxmh\"\n  \"ts\"=\"2026-03-19 04:21:51.830715\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-56\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-19 04:21:51.980503\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-56\" \"deployment\"=\"msi-deployment-56\"\n  \"ts\"=\"2026-03-19 04:21:51.996787\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-ggrxmh\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n  \"ts\"=\"2026-03-19 04:21:52.175678\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-9\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-19 04:21:52.295989\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-ggrxmh\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-lzd56b\"\n  \"ts\"=\"2026-03-19 04:21:52.334633\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-9\" \"deployment\"=\"msi-deployment-9\"\n  \"ts\"=\"2026-03-19 04:21:52.554390\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-149\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-19 04:21:52.597003\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-ggrxmh\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-03-19 04:21:52.680006\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-149\" \"deployment\"=\"msi-deployment-149\"\n  \"ts\"=\"2026-03-19 04:21:52.958897\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-ggrxmh\" \"deployment\"=\"customer-infra-basic-hcp-cluster-qpcr94\"\n  \"ts\"=\"2026-03-19 04:21:53.267438\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-19 04:21:53.267585\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-19 04:21:53.267718\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-ggrxmh\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n284\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func15.2({0x7fe3c01a6a60, 0xc00085a750})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n                | \n                | By(\"waiting for first cluster to complete creation\")\n                \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n                | \tFrequency: framework.StandardPollInterval,\n                | })\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x827b940?, 0xc00085a750?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 99\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/19/26 04:21:45.455\n  \"ts\"=\"2026-03-19 04:21:45.455935\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-19 04:21:45.456006\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-qpcr94\" \"resourceGroup\"=\"customer-rg-ggrxmh\"\n  \"ts\"=\"2026-03-19 04:21:45.775140\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-qpcr94\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-19 04:21:48.084018\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-lzd56b\" \"resourceGroup\"=\"customer-rg-ggrxmh\"\n  \"ts\"=\"2026-03-19 04:21:48.957488\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-lzd56b\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-19 04:21:51.648746\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-19 04:21:51.648859\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-19 04:21:51.648920\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-19 04:21:51.650029\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-9\"\n  \"ts\"=\"2026-03-19 04:21:51.650042\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-56\"\n  \"ts\"=\"2026-03-19 04:21:51.650081\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-149\"\n  \"ts\"=\"2026-03-19 04:21:51.650073\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-ggrxmh\"\n  \"ts\"=\"2026-03-19 04:21:51.830715\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-56\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-19 04:21:51.980503\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-56\" \"deployment\"=\"msi-deployment-56\"\n  \"ts\"=\"2026-03-19 04:21:51.996787\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-ggrxmh\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n  \"ts\"=\"2026-03-19 04:21:52.175678\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-9\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-19 04:21:52.295989\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-ggrxmh\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-lzd56b\"\n  \"ts\"=\"2026-03-19 04:21:52.334633\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-9\" \"deployment\"=\"msi-deployment-9\"\n  \"ts\"=\"2026-03-19 04:21:52.554390\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-149\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-19 04:21:52.597003\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-ggrxmh\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-03-19 04:21:52.680006\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-149\" \"deployment\"=\"msi-deployment-149\"\n  \"ts\"=\"2026-03-19 04:21:52.958897\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-ggrxmh\" \"deployment\"=\"customer-infra-basic-hcp-cluster-qpcr94\"\n  \"ts\"=\"2026-03-19 04:21:53.267438\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-19 04:21:53.267585\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-19 04:21:53.267718\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-ggrxmh\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","\u003e github.com/Azure/ARO-HCP/test/e2e.init.func15.2({0x7f54ec580e60, 0xc0009fb7d0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n                | \n                | By(\"waiting for first cluster to complete creation\")\n                \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n                | \tFrequency: framework.StandardPollInterval,\n                | })\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x827b940?, 0xc0009fb7d0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 84\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/18/26 23:20:55.083\n  \"ts\"=\"2026-03-18 23:20:55.083532\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-18 23:20:55.083598\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-kjjpjb\" \"resourceGroup\"=\"customer-rg-2hmq5s\"\n  \"ts\"=\"2026-03-18 23:20:55.333961\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-kjjpjb\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 23:20:56.369508\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-wlxsb8\" \"resourceGroup\"=\"customer-rg-2hmq5s\"\n  \"ts\"=\"2026-03-18 23:20:56.519635\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-wlxsb8\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 23:20:57.115820\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-18 23:20:57.115955\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-18 23:20:57.116022\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-18 23:20:57.117126\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-26\"\n  \"ts\"=\"2026-03-18 23:20:57.117146\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-110\"\n  \"ts\"=\"2026-03-18 23:20:57.117168\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-2hmq5s\"\n  \"ts\"=\"2026-03-18 23:20:57.117208\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-36\"\n  \"ts\"=\"2026-03-18 23:20:57.264846\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-110\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 23:20:57.327570\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-36\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 23:20:57.396893\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-110\" \"deployment\"=\"msi-deployment-110\"\n  \"ts\"=\"2026-03-18 23:20:57.432057\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-2hmq5s\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n  \"ts\"=\"2026-03-18 23:20:57.548409\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-36\" \"deployment\"=\"msi-deployment-36\"\n  \"ts\"=\"2026-03-18 23:20:57.629848\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-2hmq5s\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-wlxsb8\"\n  \"ts\"=\"2026-03-18 23:20:57.773291\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-26\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 23:20:57.819000\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-2hmq5s\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-03-18 23:20:57.904929\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-2hmq5s\" \"deployment\"=\"customer-infra-basic-hcp-cluster-kjjpjb\"\n  \"ts\"=\"2026-03-18 23:20:58.268158\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-26\" \"deployment\"=\"msi-deployment-26\"\n  \"ts\"=\"2026-03-18 23:20:58.435912\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-18 23:20:58.436063\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-18 23:20:58.436192\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-2hmq5s\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n\u003e github.com/Azure/ARO-HCP/test/e2e.init.func15.2({0x7f54ec580e60, 0xc0009fb7d0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n                | \n                | By(\"waiting for first cluster to complete creation\")\n                \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n                | \tFrequency: framework.StandardPollInterval,\n                | })\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x827b940?, 0xc0009fb7d0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 84\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/18/26 23:20:55.083\n  \"ts\"=\"2026-03-18 23:20:55.083532\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-18 23:20:55.083598\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-kjjpjb\" \"resourceGroup\"=\"customer-rg-2hmq5s\"\n  \"ts\"=\"2026-03-18 23:20:55.333961\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-kjjpjb\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 23:20:56.369508\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-wlxsb8\" \"resourceGroup\"=\"customer-rg-2hmq5s\"\n  \"ts\"=\"2026-03-18 23:20:56.519635\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-wlxsb8\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 23:20:57.115820\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-18 23:20:57.115955\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-18 23:20:57.116022\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-18 23:20:57.117126\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-26\"\n  \"ts\"=\"2026-03-18 23:20:57.117146\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-110\"\n  \"ts\"=\"2026-03-18 23:20:57.117168\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-2hmq5s\"\n  \"ts\"=\"2026-03-18 23:20:57.117208\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-36\"\n  \"ts\"=\"2026-03-18 23:20:57.264846\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-110\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 23:20:57.327570\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-36\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 23:20:57.396893\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-110\" \"deployment\"=\"msi-deployment-110\"\n  \"ts\"=\"2026-03-18 23:20:57.432057\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-2hmq5s\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n  \"ts\"=\"2026-03-18 23:20:57.548409\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-36\" \"deployment\"=\"msi-deployment-36\"\n  \"ts\"=\"2026-03-18 23:20:57.629848\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-2hmq5s\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-wlxsb8\"\n  \"ts\"=\"2026-03-18 23:20:57.773291\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-26\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 23:20:57.819000\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-2hmq5s\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-03-18 23:20:57.904929\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-2hmq5s\" \"deployment\"=\"customer-infra-basic-hcp-cluster-kjjpjb\"\n  \"ts\"=\"2026-03-18 23:20:58.268158\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-26\" \"deployment\"=\"msi-deployment-26\"\n  \"ts\"=\"2026-03-18 23:20:58.435912\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-18 23:20:58.436063\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-18 23:20:58.436192\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-2hmq5s\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034462037236518912","occurred_at":"2026-03-19T02:48:42Z","signature_id":"50eaf54c62b2e66baf8c41c420a48be01d18df270046efa2cd684e3bd695dfbd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"bae0429ac1980489e8f968319dc031c07dd05138b989654b348f26287e5c313e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"893592381c30302c0ad58ee9d8f6dc46d73953b616076e02ee3cc8d0f8a1abef","failure_pattern":"ERROR CODE: AuthorizationFailed","search_query":"ERROR CODE: AuthorizationFailed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:81]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001504fc0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-basic-hcp-cluster-7hz8zk\" in resourcegroup=\"customer-rg-clsh4k\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609\n    --------------------------------------------------------------------------------\n    RESPONSE 403: 403 Forbidden\n    ERROR CODE: AuthorizationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"AuthorizationFailed\",\n        \"message\": \"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Resources/deployments/operationStatuses/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609' or the scope is invalid. If access was recently granted, please refresh your credentials.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-basic-hcp-cluster-7hz8zk\\\" in resourcegroup=\\\"customer-rg-clsh4k\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Resources/deployments/operationStatuses/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc001504fa0\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-basic-hcp-cluster-7hz8zk\\\" in resourcegroup=\\\"customer-rg-clsh4k\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Resources/deployments/operationStatuses/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0014bb1a0\u003e{\n                ErrorCode: \"AuthorizationFailed\",\n                StatusCode: 403,\n                RawResponse: {\n                    Status: \"403 Forbidden\",\n                    StatusCode: 403,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Failure-Cause\": [\"gateway\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"06490acc-bbf6-4428-8144-e61db004ace3\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"EASTUS2EUAP:20260316T060848Z:06490acc-bbf6-4428-8144-e61db004ace3\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"569\"],\n                        \"Content-Type\": [\n                            \"applic...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:81]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001504fc0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-basic-hcp-cluster-7hz8zk\" in resourcegroup=\"customer-rg-clsh4k\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609\n    --------------------------------------------------------------------------------\n    RESPONSE 403: 403 Forbidden\n    ERROR CODE: AuthorizationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"AuthorizationFailed\",\n        \"message\": \"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Resources/deployments/operationStatuses/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609' or the scope is invalid. If access was recently granted, please refresh your credentials.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-basic-hcp-cluster-7hz8zk\\\" in resourcegroup=\\\"customer-rg-clsh4k\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Resources/deployments/operationStatuses/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc001504fa0\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-basic-hcp-cluster-7hz8zk\\\" in resourcegroup=\\\"customer-rg-clsh4k\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Resources/deployments/operationStatuses/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0014bb1a0\u003e{\n                ErrorCode: \"AuthorizationFailed\",\n                StatusCode: 403,\n                RawResponse: {\n                    Status: \"403 Forbidden\",\n                    StatusCode: 403,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Failure-Cause\": [\"gateway\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"06490acc-bbf6-4428-8144-e61db004ace3\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"EASTUS2EUAP:20260316T060848Z:06490acc-bbf6-4428-8144-e61db004ace3\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"569\"],\n                        \"Content-Type\": [\n                            \"applic...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"f5fb0b3b4bff165498e2d9ddacd8e71b63b31ca295b435d5432bfc3cf1966be7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"eff4ac5ab820367904c1fb3e78845caba542fd4825dfa1c42a1fdcf6260aa246","failure_pattern":"ERROR CODE: AuthorizationFailed; detail message The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '\u003cuuid\u003e' does not have authorization to perform action 'Microsoft.RedHatOpenShift/hcpOpenShiftClusters/\u003cresource\u003e' over scope '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128' or the scope is invalid. If access was recently granted, please refresh your credentials.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: AuthorizationFailed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0008f2c20\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-rkr4r8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n    --------------------------------------------------------------------------------\n    RESPONSE 403: 403 Forbidden\n    ERROR CODE: AuthorizationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"AuthorizationFailed\",\n        \"message\": \"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.RedHatOpenShift/hcpOpenShiftClusters/write' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128' or the scope is invalid. If access was recently granted, please refresh your credentials.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-rkr4r8\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.RedHatOpenShift/hcpOpenShiftClusters/write' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0008f2c00\u003e{\n            msg: \"failed starting cluster creation \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-rkr4r8\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.RedHatOpenShift/hcpOpenShiftClusters/write' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0014046f0\u003e{\n                ErrorCode: \"AuthorizationFailed\",\n                StatusCode: 403,\n                RawResponse: {\n                    Status: \"403 Forbidden\",\n                    StatusCode: 403,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Length\": [\"523\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"2935921b-184b-4fc2-9c09-4c26121abf36\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Date\": [\n                            \"Mon, 16 Mar 2026 05:50:31 GMT\",\n                        ],\n                        \"Content-Type\": [\n                            \"application/json; charset=utf-8\",\n                        ],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Failure-Cause\": [\"gateway\"],\n                        \"X-Ms-Request-Id\": [\n                            \"2935921b-184b-4fc2-9c09-4c26121abf36\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"EASTUS2EUAP:20260316T055032Z:2935921b-184b-4fc2-9c09-4c26121abf36\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n    ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0008f2c20\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-rkr4r8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n    --------------------------------------------------------------------------------\n    RESPONSE 403: 403 Forbidden\n    ERROR CODE: AuthorizationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"AuthorizationFailed\",\n        \"message\": \"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.RedHatOpenShift/hcpOpenShiftClusters/write' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128' or the scope is invalid. If access was recently granted, please refresh your credentials.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-rkr4r8\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.RedHatOpenShift/hcpOpenShiftClusters/write' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0008f2c00\u003e{\n            msg: \"failed starting cluster creation \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-rkr4r8\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.RedHatOpenShift/hcpOpenShiftClusters/write' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0014046f0\u003e{\n                ErrorCode: \"AuthorizationFailed\",\n                StatusCode: 403,\n                RawResponse: {\n                    Status: \"403 Forbidden\",\n                    StatusCode: 403,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Length\": [\"523\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"2935921b-184b-4fc2-9c09-4c26121abf36\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Date\": [\n                            \"Mon, 16 Mar 2026 05:50:31 GMT\",\n                        ],\n                        \"Content-Type\": [\n                            \"application/json; charset=utf-8\",\n                        ],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Failure-Cause\": [\"gateway\"],\n                        \"X-Ms-Request-Id\": [\n                            \"2935921b-184b-4fc2-9c09-4c26121abf36\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"EASTUS2EUAP:20260316T055032Z:2935921b-184b-4fc2-9c09-4c26121abf36\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n    ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"db956ec4e40e35159a8af8607f32da52079e787390ee2e9a142f0fef2f1112c8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"3efdb49df868badec71fe7b3172881c5f208bc6b8873fb2baedf7ca51b8593a4","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AuthorizationFailed; detail message The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '\u003cuuid\u003e' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/\u003cresource\u003e' over scope '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0004993c0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-ea-cluster-lqkgx2\" in resourcegroup=\"external-auth-cluster-rhfph2\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Resources/deployments/customer-infra-ea-cluster-lqkgx2/operationStatuses/08584279665118667979\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Forbidden\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\r\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-ea-cluster-lqkgx2\\\" in resourcegroup=\\\"external-auth-cluster-rhfph2\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Resources/deployments/customer-infra-ea-cluster-lqkgx2/operationStatuses/08584279665118667979\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000499380\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-ea-cluster-lqkgx2\\\" in resourcegroup=\\\"external-auth-cluster-rhfph2\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Resources/deployments/customer-infra-ea-cluster-lqkgx2/operationStatuses/08584279665118667979\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000baa6f0\u003e{\n                ErrorCode: \"DeploymentFailed\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n     ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0004993c0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-ea-cluster-lqkgx2\" in resourcegroup=\"external-auth-cluster-rhfph2\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Resources/deployments/customer-infra-ea-cluster-lqkgx2/operationStatuses/08584279665118667979\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Forbidden\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\r\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-ea-cluster-lqkgx2\\\" in resourcegroup=\\\"external-auth-cluster-rhfph2\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Resources/deployments/customer-infra-ea-cluster-lqkgx2/operationStatuses/08584279665118667979\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000499380\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-ea-cluster-lqkgx2\\\" in resourcegroup=\\\"external-auth-cluster-rhfph2\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Resources/deployments/customer-infra-ea-cluster-lqkgx2/operationStatuses/08584279665118667979\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000baa6f0\u003e{\n                ErrorCode: \"DeploymentFailed\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n     ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"a10960aad13ed9377a917637cf1712dc7c65ca2dd34b3f121b55ef5f9935f1e0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1ab4a070713c2799384044a1c8b07d256e44b43cd64be86c9bc5395d9df5c28e","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AuthorizationFailed; detail message The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '\u003cuuid\u003e' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/\u003cresource\u003e' over scope '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/virtualNetworks/customer-vnet' or the scope is invalid. If access was recently granted, please refresh your credentials.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:74]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00093d060\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-np-autoscale-cluster-5pp8qh\" in resourcegroup=\"np-autoscaling-l7brcq\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Resources/deployments/customer-infra-np-autoscale-cluster-5pp8qh/operationStatuses/08584279658982549597\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Forbidden\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\r\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Network/virtualNetworks/customer-vnet' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-np-autoscale-cluster-5pp8qh\\\" in resourcegroup=\\\"np-autoscaling-l7brcq\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Resources/deployments/customer-infra-np-autoscale-cluster-5pp8qh/operationStatuses/08584279658982549597\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Network/virtualNetworks/customer-vnet' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00093d020\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-np-autoscale-cluster-5pp8qh\\\" in resourcegroup=\\\"np-autoscaling-l7brcq\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Resources/deployments/customer-infra-np-autoscale-cluster-5pp8qh/operationStatuses/08584279658982549597\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Network/virtualNetworks/customer-vnet' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0008efc50\u003e{\n                ErrorCode: \"DeploymentFailed\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: ED37EB9E1FA645FA954684A176851865 Ref B: CH1AA2020620045 Ref C: 2026-03-16T06:05:39Z\",\n                        ],\n                        \"Date\": [\n        ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:74]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00093d060\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-np-autoscale-cluster-5pp8qh\" in resourcegroup=\"np-autoscaling-l7brcq\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Resources/deployments/customer-infra-np-autoscale-cluster-5pp8qh/operationStatuses/08584279658982549597\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Forbidden\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\r\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Network/virtualNetworks/customer-vnet' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-np-autoscale-cluster-5pp8qh\\\" in resourcegroup=\\\"np-autoscaling-l7brcq\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Resources/deployments/customer-infra-np-autoscale-cluster-5pp8qh/operationStatuses/08584279658982549597\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Network/virtualNetworks/customer-vnet' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00093d020\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-np-autoscale-cluster-5pp8qh\\\" in resourcegroup=\\\"np-autoscaling-l7brcq\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Resources/deployments/customer-infra-np-autoscale-cluster-5pp8qh/operationStatuses/08584279658982549597\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Network/virtualNetworks/customer-vnet' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0008efc50\u003e{\n                ErrorCode: \"DeploymentFailed\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: ED37EB9E1FA645FA954684A176851865 Ref B: CH1AA2020620045 Ref C: 2026-03-16T06:05:39Z\",\n                        ],\n                        \"Date\": [\n        ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"aa224806ca603d56173bc8c4551b5efd8e98c6c2cdc24d4ee7cb97d7e3e4866e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6461ea09e6946cd19f6c887ff25cabbd85071eb5cb02d829439f92ba37e09b36","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AuthorizationFailed; detail message The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '\u003cuuid\u003e' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/\u003cresource\u003e' over scope '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1' or the scope is invalid. If access was recently granted, please refresh your credentials.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:96]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000760480\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-basic-cluster-wk5cgh\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-dvvbjq\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Resources/deployments/customer-infra-basic-cluster-wk5cgh/operationStatuses/08584279668585153729\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Forbidden\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\r\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-basic-cluster-wk5cgh\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-dvvbjq\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Resources/deployments/customer-infra-basic-cluster-wk5cgh/operationStatuses/08584279668585153729\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000760460\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-basic-cluster-wk5cgh\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-dvvbjq\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Resources/deployments/customer-infra-basic-cluster-wk5cgh/operationStatuses/08584279668585153729\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0012bd110\u003e{\n                ErrorCode: \"DeploymentFailed\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Length\": [\"839\"],\n                 ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:96]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000760480\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-basic-cluster-wk5cgh\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-dvvbjq\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Resources/deployments/customer-infra-basic-cluster-wk5cgh/operationStatuses/08584279668585153729\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Forbidden\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\r\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-basic-cluster-wk5cgh\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-dvvbjq\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Resources/deployments/customer-infra-basic-cluster-wk5cgh/operationStatuses/08584279668585153729\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000760460\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-basic-cluster-wk5cgh\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-dvvbjq\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Resources/deployments/customer-infra-basic-cluster-wk5cgh/operationStatuses/08584279668585153729\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0012bd110\u003e{\n                ErrorCode: \"DeploymentFailed\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Length\": [\"839\"],\n                 ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","signature_id":"27453cdf92c9666ede34c49454a77bb28518b6f6a37d519d92e216ae52bf28bc","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c53d05de98c210ebd74e4692ef2a5929f57c23f85cc96f599152530f1cb419d3","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:81]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0016053a0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster-9d7jkm\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-9d7jkm/operationStatuses/08584276114880859088\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-jg7phn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-eqbgltuyzhpg4\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260320T083028Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'a7739b02-f190-492e-b6ca-5904c97135c6', request correlation id '6c19fc94-efec-4cab-8d4f-81ceaf7a1471'.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260320T083027Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '0ec4bb4f-3c7a-4a85-9ef0-62497151a6f0', request correlation id '6c19fc94-efec-4cab-8d4f-81ceaf7a1471'.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260320T083027Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '5fcbd0fd-49b3-4339-b567-de1c1596aaa0', request correlation id '6c19fc94-efec-4cab-8d4f-81ceaf7a1471'.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260320T083027Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '92858e8c-aa82-4d12-af4e-bbaf740beedc', request correlation id '6c19fc94-efec-4cab-8d4f-81ceaf7a1471'.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260320T083026Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '5aedb7a4-fd70-4e59-8a0f-c36fcb99fe4a', request correlation id '6c19fc94-efec-4cab-8d4f-81ceaf7a1471'.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260320T083026Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '6ad7864e-528a-4bd7-a91c-c7eead01227a', request correlation id '6c19fc94-efec-4cab-8d4f-81ceaf7a1471'.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260320T083025Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'b474b493-2d00-4cd4-8b16-04c1849437da', request correlation id '6c19fc94-efec-4cab-8d4f-81ceaf7a1471'.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260320T083024Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'e178a66f-7a87-4e67-8562-6aafb6036181', request correlation id '6c19fc94-efec-4cab-8d4f-81ceaf7a1471'.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260320T083023Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'e59eefa1-b6f0-4d97-8d61-9506dc6c5ff4', request correlation id '6c19fc94-efec-4cab-8d4f-81ceaf7a1471'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-basic-hcp-cluster-9d7jkm\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-9d7jkm/operationStatuses/08584276114880859088\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-jg7phn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-eqbgltuyzhpg4\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260320T083028Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'a7739b02-f190-492e-b6ca-5904c97135c6', request correlation id '6c19fc94-efec-4cab-8d4f-81ceaf7a1471'.\\\\\\\"\\\\r\\\\n          },\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260320T083027Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '0ec4bb4f-3c7a-4a85-9ef0-62497151a6f0', request correlation id '6c19fc94-efec-4cab-8d4f-81ceaf7a1471'.\\\\\\\"\\\\r\\\\n          },\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260320T083027Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '5fcbd0fd-49b3-4339-b567-de1c1596aaa0', request correlation id '6c19fc94-efec-4cab-8d4f-81ceaf7a1471'.\\\\\\\"\\\\r\\\\n          },\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"En...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:81]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0016053a0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster-9d7jkm\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-9d7jkm/operationStatuses/08584276114880859088\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-jg7phn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-eqbgltuyzhpg4\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260320T083028Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'a7739b02-f190-492e-b6ca-5904c97135c6', request correlation id '6c19fc94-efec-4cab-8d4f-81ceaf7a1471'.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260320T083027Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '0ec4bb4f-3c7a-4a85-9ef0-62497151a6f0', request correlation id '6c19fc94-efec-4cab-8d4f-81ceaf7a1471'.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260320T083027Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '5fcbd0fd-49b3-4339-b567-de1c1596aaa0', request correlation id '6c19fc94-efec-4cab-8d4f-81ceaf7a1471'.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260320T083027Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '92858e8c-aa82-4d12-af4e-bbaf740beedc', request correlation id '6c19fc94-efec-4cab-8d4f-81ceaf7a1471'.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-controller-manager\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260320T083026Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '5aedb7a4-fd70-4e59-8a0f-c36fcb99fe4a', request correlation id '6c19fc94-efec-4cab-8d4f-81ceaf7a1471'.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260320T083026Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '6ad7864e-528a-4bd7-a91c-c7eead01227a', request correlation id '6c19fc94-efec-4cab-8d4f-81ceaf7a1471'.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/service\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260320T083025Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'b474b493-2d00-4cd4-8b16-04c1849437da', request correlation id '6c19fc94-efec-4cab-8d4f-81ceaf7a1471'.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/file-csi-driver\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260320T083024Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'e178a66f-7a87-4e67-8562-6aafb6036181', request correlation id '6c19fc94-efec-4cab-8d4f-81ceaf7a1471'.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n            \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/kms\\\",\\r\\n            \\\"message\\\": \\\"Encountered internal server error. Diagnostic information: timestamp '20260320T083023Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'e59eefa1-b6f0-4d97-8d61-9506dc6c5ff4', request correlation id '6c19fc94-efec-4cab-8d4f-81ceaf7a1471'.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-basic-hcp-cluster-9d7jkm\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster-9d7jkm/operationStatuses/08584276114880859088\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-jg7phn/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-eqbgltuyzhpg4\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260320T083028Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id 'a7739b02-f190-492e-b6ca-5904c97135c6', request correlation id '6c19fc94-efec-4cab-8d4f-81ceaf7a1471'.\\\\\\\"\\\\r\\\\n          },\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260320T083027Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '0ec4bb4f-3c7a-4a85-9ef0-62497151a6f0', request correlation id '6c19fc94-efec-4cab-8d4f-81ceaf7a1471'.\\\\\\\"\\\\r\\\\n          },\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/control-plane\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"Encountered internal server error. Diagnostic information: timestamp '20260320T083027Z', subscription id 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', tracking id '5fcbd0fd-49b3-4339-b567-de1c1596aaa0', request correlation id '6c19fc94-efec-4cab-8d4f-81ceaf7a1471'.\\\\\\\"\\\\r\\\\n          },\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-107/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cloud-network-config\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"En...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034909477634838528","occurred_at":"2026-03-20T08:26:40Z","signature_id":"22da5bd587bc767d607c5a4f6c8abb3889862b670ce2972f3c2193f78bff31d6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ca7cc2da32e290afc8b2a3a18285c29643093d3821378dbf18bf57bc20cad012","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message ClientCertificateCredential: unable to resolve an endpoint: server response error: context deadline exceeded; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001828480\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-99xb7j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\",\n      \"name\": \"6e36926c-e6c5-49f7-a607-9f56a4bbb73b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:05:01.130113225Z\",\n      \"endTime\": \"2026-03-18T22:06:29.307118365Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Failed to verify existence of managed resource group 'patch-name-99xb7j--managed': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"failed to get quota usage data: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-99xb7j\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\\",\\n  \\\"name\\\": \\\"6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:05:01.130113225Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:06:29.307118365Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to verify existence of managed resource group 'patch-name-99xb7j--managed': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"failed to get quota usage data: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc001828460\u003e{\n            msg: \"failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-99xb7j\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\\",\\n  \\\"name\\\": \\\"6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:05:01.130113225Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:06:29.307118365Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to verify existence of managed resource group 'patch-name-99xb7j--managed': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerErro...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001828480\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-99xb7j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\",\n      \"name\": \"6e36926c-e6c5-49f7-a607-9f56a4bbb73b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:05:01.130113225Z\",\n      \"endTime\": \"2026-03-18T22:06:29.307118365Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Failed to verify existence of managed resource group 'patch-name-99xb7j--managed': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"failed to get quota usage data: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-99xb7j\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\\",\\n  \\\"name\\\": \\\"6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:05:01.130113225Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:06:29.307118365Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to verify existence of managed resource group 'patch-name-99xb7j--managed': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"failed to get quota usage data: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc001828460\u003e{\n            msg: \"failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-99xb7j\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\\",\\n  \\\"name\\\": \\\"6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:05:01.130113225Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:06:29.307118365Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to verify existence of managed resource group 'patch-name-99xb7j--managed': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerErro...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"237ff6b17b8ec98ad232c04a0ed44127d05d48dc83399baccc02c34cebfbfc12","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ca156a6e6dc55851f861662371785d03e87c4fa68946334f3057fa599843ab7e","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00133a3c0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-sv2zcp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\",\n      \"name\": \"b06360bf-0a54-4e7d-b9f1-3e5b6626253e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:17:00.823479898Z\",\n      \"endTime\": \"2026-03-18T22:18:09.152905722Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"insufficient public IP address quota: required 2, available 0\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Failed to get virtual network 'customer-vnet-name' details: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-120/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"failed to get network security group 'customer-nsg-name' details: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-sv2zcp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\\",\\n  \\\"name\\\": \\\"b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:17:00.823479898Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:18:09.152905722Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-name' details: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-120/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00133a3a0\u003e{\n            msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-sv2zcp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\\",\\n  \\\"name\\\": \\\"b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:17:00.823479898Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:18:09.152905722Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-name' details: Cl...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00133a3c0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-sv2zcp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\",\n      \"name\": \"b06360bf-0a54-4e7d-b9f1-3e5b6626253e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:17:00.823479898Z\",\n      \"endTime\": \"2026-03-18T22:18:09.152905722Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"insufficient public IP address quota: required 2, available 0\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Failed to get virtual network 'customer-vnet-name' details: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-120/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"failed to get network security group 'customer-nsg-name' details: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-sv2zcp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\\",\\n  \\\"name\\\": \\\"b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:17:00.823479898Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:18:09.152905722Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-name' details: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-120/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00133a3a0\u003e{\n            msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-sv2zcp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\\",\\n  \\\"name\\\": \\\"b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:17:00.823479898Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:18:09.152905722Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-name' details: Cl...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"fd62b1986336789b07747b1dea7d2f2f4649096a7807af24263707e890c6d458","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"7af77482c75e06f47d745495b22aecd555e8fcdf64f2c7cf43f5cafdb158f0dd","failure_pattern":"ERROR CODE: ResourceGroupDeletionFailed","search_query":"ERROR CODE: ResourceGroupDeletionFailed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-09"],"prior_runs_affected":4,"prior_last_seen_at":"2026-03-12T02:54:14Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n    \u003c*errors.joinError | 0xc000a84948\u003e: \n    failed to cleanup resource group: failed waiting for resourcegroup=\"np-autoscaling-hsrbbh\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/operationresults/eyJqb2JJZCI6IlJFU09VUkNFR1JPVVBERUxFVElPTkpPQi1OUDoyREFVVE9TQ0FMSU5HOjJESFNSQkJILUVBU1RVUzIiLCJqb2JMb2NhdGlvbiI6ImVhc3R1czIifQ\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: ResourceGroupDeletionFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceGroupDeletionFailed\",\n        \"message\": \"The deletion of resource group 'np-autoscaling-hsrbbh' did not finish within allowed time. Resource group provisioning state rolled back. Please check audit logs for more details.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0006c30a0\u003e{\n                msg: \"failed to cleanup resource group: failed waiting for resourcegroup=\\\"np-autoscaling-hsrbbh\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/operationresults/eyJqb2JJZCI6IlJFU09VUkNFR1JPVVBERUxFVElPTkpPQi1OUDoyREFVVE9TQ0FMSU5HOjJESFNSQkJILUVBU1RVUzIiLCJqb2JMb2NhdGlvbiI6ImVhc3R1czIifQ\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: ResourceGroupDeletionFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ResourceGroupDeletionFailed\\\",\\n    \\\"message\\\": \\\"The deletion of resource group 'np-autoscaling-hsrbbh' did not finish within allowed time. Resource group provisioning state rolled back. Please check audit logs for more details.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                err: \u003c*fmt.wrapError | 0xc0006c3080\u003e{\n                    msg: \"failed waiting for resourcegroup=\\\"np-autoscaling-hsrbbh\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/operationresults/eyJqb2JJZCI6IlJFU09VUkNFR1JPVVBERUxFVElPTkpPQi1OUDoyREFVVE9TQ0FMSU5HOjJESFNSQkJILUVBU1RVUzIiLCJqb2JMb2NhdGlvbiI6ImVhc3R1czIifQ\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: ResourceGroupDeletionFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ResourceGroupDeletionFailed\\\",\\n    \\\"message\\\": \\\"The deletion of resource group 'np-autoscaling-hsrbbh' did not finish within allowed time. Resource group provisioning state rolled back. Please check audit logs for more details.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                    err: \u003c*exported.ResponseError | 0xc00092b0b0\u003e{\n                        ErrorCode: \"ResourceGroupDeletionFailed\",\n                        StatusCode: 409,\n                        RawResponse: {\n                            Status: \"409 Conflict\",\n                            StatusCode: 409,\n                            Proto: \"HTTP/2.0\",\n                            ProtoMajor: 2,\n                            ProtoMinor: 0,\n                            Header: {\n                                \"X-Ms-Routing-Request-Id\": [\n                                    \"CANADACENTRAL:20260319T034906Z:75f74363-b48d-43af-b448-6066e54e30d1\",\n                                ],\n                                \"Content-Length\": [\"240\"],\n                                \"Content-Type\": [\n                                    \"application/json; charset=utf-8\",\n                                ],\n                                \"Expires\": [\"-1\"],\n                                \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                                \"X-Ms-Request-Id\": [\n                                    \"75f74363-b48d-43af-b448-6066e54e30d1\",\n                                ],\n                                \"Date\": [\n                                    \"Thu, 19 Mar 2026 03:49:05 GMT\",\n                                ],\n                                \"Cache-Control\": [\"no-cache\"],\n                                \"Strict-Transport-Security\": [\n                                    \"max-age=31536000; includeSubDomains\",\n                                ],\n                                \"X-Content-Type-Options\": [\"nosniff\"],\n                                \"Pragma\": [\"no-cache\"],\n                                \"X-Ms-Correlation-Request-Id\": [\n                                    \"75f74363-b48d-43af-b448-6066e54e30d1\",\n                                ],\n                                \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                                \"X-Msedge-Ref...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n    \u003c*errors.joinError | 0xc000a84948\u003e: \n    failed to cleanup resource group: failed waiting for resourcegroup=\"np-autoscaling-hsrbbh\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/operationresults/eyJqb2JJZCI6IlJFU09VUkNFR1JPVVBERUxFVElPTkpPQi1OUDoyREFVVE9TQ0FMSU5HOjJESFNSQkJILUVBU1RVUzIiLCJqb2JMb2NhdGlvbiI6ImVhc3R1czIifQ\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: ResourceGroupDeletionFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceGroupDeletionFailed\",\n        \"message\": \"The deletion of resource group 'np-autoscaling-hsrbbh' did not finish within allowed time. Resource group provisioning state rolled back. Please check audit logs for more details.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        errs: [\n            \u003c*fmt.wrapError | 0xc0006c30a0\u003e{\n                msg: \"failed to cleanup resource group: failed waiting for resourcegroup=\\\"np-autoscaling-hsrbbh\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/operationresults/eyJqb2JJZCI6IlJFU09VUkNFR1JPVVBERUxFVElPTkpPQi1OUDoyREFVVE9TQ0FMSU5HOjJESFNSQkJILUVBU1RVUzIiLCJqb2JMb2NhdGlvbiI6ImVhc3R1czIifQ\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: ResourceGroupDeletionFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ResourceGroupDeletionFailed\\\",\\n    \\\"message\\\": \\\"The deletion of resource group 'np-autoscaling-hsrbbh' did not finish within allowed time. Resource group provisioning state rolled back. Please check audit logs for more details.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                err: \u003c*fmt.wrapError | 0xc0006c3080\u003e{\n                    msg: \"failed waiting for resourcegroup=\\\"np-autoscaling-hsrbbh\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/operationresults/eyJqb2JJZCI6IlJFU09VUkNFR1JPVVBERUxFVElPTkpPQi1OUDoyREFVVE9TQ0FMSU5HOjJESFNSQkJILUVBU1RVUzIiLCJqb2JMb2NhdGlvbiI6ImVhc3R1czIifQ\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: ResourceGroupDeletionFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ResourceGroupDeletionFailed\\\",\\n    \\\"message\\\": \\\"The deletion of resource group 'np-autoscaling-hsrbbh' did not finish within allowed time. Resource group provisioning state rolled back. Please check audit logs for more details.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                    err: \u003c*exported.ResponseError | 0xc00092b0b0\u003e{\n                        ErrorCode: \"ResourceGroupDeletionFailed\",\n                        StatusCode: 409,\n                        RawResponse: {\n                            Status: \"409 Conflict\",\n                            StatusCode: 409,\n                            Proto: \"HTTP/2.0\",\n                            ProtoMajor: 2,\n                            ProtoMinor: 0,\n                            Header: {\n                                \"X-Ms-Routing-Request-Id\": [\n                                    \"CANADACENTRAL:20260319T034906Z:75f74363-b48d-43af-b448-6066e54e30d1\",\n                                ],\n                                \"Content-Length\": [\"240\"],\n                                \"Content-Type\": [\n                                    \"application/json; charset=utf-8\",\n                                ],\n                                \"Expires\": [\"-1\"],\n                                \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                                \"X-Ms-Request-Id\": [\n                                    \"75f74363-b48d-43af-b448-6066e54e30d1\",\n                                ],\n                                \"Date\": [\n                                    \"Thu, 19 Mar 2026 03:49:05 GMT\",\n                                ],\n                                \"Cache-Control\": [\"no-cache\"],\n                                \"Strict-Transport-Security\": [\n                                    \"max-age=31536000; includeSubDomains\",\n                                ],\n                                \"X-Content-Type-Options\": [\"nosniff\"],\n                                \"Pragma\": [\"no-cache\"],\n                                \"X-Ms-Correlation-Request-Id\": [\n                                    \"75f74363-b48d-43af-b448-6066e54e30d1\",\n                                ],\n                                \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                                \"X-Msedge-Ref...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034461239047884800","occurred_at":"2026-03-19T02:45:32Z","signature_id":"a93664b0f8de43168835d2de9815949d52e2a6629033cdd0b70a42e2ec444dd4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":38,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":3,"prior_window_starts":["2026-02-23","2026-03-02","2026-03-09"],"prior_runs_affected":14,"prior_last_seen_at":"2026-03-12T00:28:00Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:47]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:47]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034462034761879552","occurred_at":"2026-03-19T02:48:42Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"946ef15ddb5ce397e2bb33699004180a31fc84de8a34d3332fb8a309352f5dcd","failure_pattern":"found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-6d7nns","search_query":"occurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n    \u003c*errors.joinError | 0xc0016cd758\u003e: \n    found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-6d7nns\n    {\n        errs: [\n            \u003c*errors.errorString | 0xc000502840\u003e{\n                s: \"found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-6d7nns\",\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n    \u003c*errors.joinError | 0xc0016cd758\u003e: \n    found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-6d7nns\n    {\n        errs: [\n            \u003c*errors.errorString | 0xc000502840\u003e{\n                s: \"found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-6d7nns\",\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034460893433040896","occurred_at":"2026-03-19T02:44:08Z","signature_id":"5c1d44e5ff7f770d70f762fbdbd716d36108ed68fa1b75889b728f7eff12eece","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"af239a3015f0e30dda0063e5e225f7586d73b93f695ac3f6f1537ef9509461a0","failure_pattern":"found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-dgq99r","search_query":"occurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012c3e30\u003e: \n    found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-dgq99r\n    {\n        errs: [\n            \u003c*errors.errorString | 0xc0017cdab0\u003e{\n                s: \"found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-dgq99r\",\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n    \u003c*errors.joinError | 0xc0012c3e30\u003e: \n    found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-dgq99r\n    {\n        errs: [\n            \u003c*errors.errorString | 0xc0017cdab0\u003e{\n                s: \"found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-dgq99r\",\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034501643403268096","occurred_at":"2026-03-19T05:26:05Z","signature_id":"445a14048bc07b639a25654ecd6e536af9a34a828a90c1ba7f3465565fb6eec9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"6f007169b9ed1259d703f7f5d2930c60b49afb43735680b984afa39d6c9536d4","failure_pattern":"found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-whz9b5","search_query":"occurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a1ebd0\u003e: \n    found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-whz9b5\n    {\n        errs: [\n            \u003c*errors.errorString | 0xc001598330\u003e{\n                s: \"found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-whz9b5\",\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:243]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a1ebd0\u003e: \n    found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-whz9b5\n    {\n        errs: [\n            \u003c*errors.errorString | 0xc001598330\u003e{\n                s: \"found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-whz9b5\",\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034572752144306176","occurred_at":"2026-03-19T10:08:39Z","signature_id":"0a13d519917f467425b7e9ea014f3718d64911f0abc0d49aa8d9f7fc6188abe9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5fe2fe981b3ddaad417bdcd1b0fe174fa1aefc5168a3276c4d05da3569ee9c7f","failure_pattern":"not to be empty","search_query":"not to be empty","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:126]: Expected\n    \u003c[]*armredhatopenshifthcp.ImageDigestMirror | len:0, cap:0\u003e: nil\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:126]: Expected\n    \u003c[]*armredhatopenshifthcp.ImageDigestMirror | len:0, cap:0\u003e: nil\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2035174783125032960","occurred_at":"2026-03-21T02:00:53Z","signature_id":"c2437d1f3013b7e5529f2ad9cba0f07c235e209120a5623fcb08599717d53a8d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"3abd48de63dd3956140a30196e0f3dc44998cdba01a694484355c666029b9eeb","failure_pattern":"subscription openshift-nfd/nfd not ready, state=\"\" (OLM still processing, no errors detected)","search_query":"subscription openshift-nfd/nfd not ready, state=\"\" (OLM still processing, no errors detected)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-03-15..2026-03-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:60]: Timed out after 600.007s.\nNFD operator should be installed successfully\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001039e40\u003e: \n    subscription openshift-nfd/nfd not ready, state=\"\" (OLM still processing, no errors detected)\n    {\n        s: \"subscription openshift-nfd/nfd not ready, state=\\\"\\\" (OLM still processing, no errors detected)\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:60]: Timed out after 600.007s.\nNFD operator should be installed successfully\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001039e40\u003e: \n    subscription openshift-nfd/nfd not ready, state=\"\" (OLM still processing, no errors detected)\n    {\n        s: \"subscription openshift-nfd/nfd not ready, state=\\\"\\\" (OLM still processing, no errors detected)\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033362659054718976","occurred_at":"2026-03-16T02:00:10Z","signature_id":"69b6d65c2a309f09c806b140114cf9bcb0384b2a18f675177e552539baa0bdd1","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}