{"meta":{"start_date":"2026-03-18","end_date":"2026-03-18","start_at":"2026-03-18T00:00:00Z","end_at":"2026-03-19T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T21:36:34Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":9,"failed_runs":2,"raw_occurrences":23,"matched_occurrences":22,"runs_affected":1},"rows":[{"environment":"prod","failure_pattern_id":"8a0eb43d82c81f5ac221f93e261d7d3906eccf9364890faef15f25844bdadb79","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","occurrences":11,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":11,"after_last_push_count":23,"trend_counts":[0,0,0,0,7,0,11],"trend_range":"2026-03-12..2026-03-18 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-02"],"prior_runs_affected":3,"prior_last_seen_at":"2026-03-04T11:18:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000962220\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-hfk97l: failed waiting for cluster=\"cluster-zstream-4-20-hfk97l\" in resourcegroup=\"rg-zstream-upgrade-4-20-6jjt78\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\",\n      \"name\": \"a535b9a7-79bf-4ae9-99b6-679319213e49\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:10:00.515143027Z\",\n      \"endTime\": \"2026-03-18T22:11:58.502121438Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-20-hfk97l: failed waiting for cluster=\\\"cluster-zstream-4-20-hfk97l\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-6jjt78\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\\\",\\n  \\\"name\\\": \\\"a535b9a7-79bf-4ae9-99b6-679319213e49\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:10:00.515143027Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:11:58.502121438Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0009621e0\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-hfk97l\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-6jjt78\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\\\",\\n  \\\"name\\\": \\\"a535b9a7-79bf-4ae9-99b6-679319213e49\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:10:00.515143027Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:11:58.502121438Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000e803c0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/a23e4d00-7c12-428e-b637-d6e87c428e63\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260318T221203Z:86a9a285-d268-4eaa-81f2-1b43b6865e4f\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"503\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"86a9a285-d268-4eaa-81f2-1b43b6865e4f\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 30110B94D9A9488595249B68A2A891EF Ref B: CHI261140505062 Ref C: 2026-03-18T22:12:03Z\",\n                        ],\n                        \"Date\": [\n                            \"Wed, 18 Mar 2026 22:12...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000962220\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-hfk97l: failed waiting for cluster=\"cluster-zstream-4-20-hfk97l\" in resourcegroup=\"rg-zstream-upgrade-4-20-6jjt78\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\",\n      \"name\": \"a535b9a7-79bf-4ae9-99b6-679319213e49\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:10:00.515143027Z\",\n      \"endTime\": \"2026-03-18T22:11:58.502121438Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-20-hfk97l: failed waiting for cluster=\\\"cluster-zstream-4-20-hfk97l\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-6jjt78\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\\\",\\n  \\\"name\\\": \\\"a535b9a7-79bf-4ae9-99b6-679319213e49\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:10:00.515143027Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:11:58.502121438Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0009621e0\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-hfk97l\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-6jjt78\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\\\",\\n  \\\"name\\\": \\\"a535b9a7-79bf-4ae9-99b6-679319213e49\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:10:00.515143027Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:11:58.502121438Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000e803c0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/a23e4d00-7c12-428e-b637-d6e87c428e63\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260318T221203Z:86a9a285-d268-4eaa-81f2-1b43b6865e4f\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"503\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"86a9a285-d268-4eaa-81f2-1b43b6865e4f\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 30110B94D9A9488595249B68A2A891EF Ref B: CHI261140505062 Ref C: 2026-03-18T22:12:03Z\",\n                        ],\n                        \"Date\": [\n                            \"Wed, 18 Mar 2026 22:12...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000a8e6a0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-fncbll\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\",\n      \"name\": \"1dcfdab9-27df-483f-a009-42303866394b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:10:00.698917174Z\",\n      \"endTime\": \"2026-03-18T22:12:23.691383966Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-fncbll\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\\\",\\n  \\\"name\\\": \\\"1dcfdab9-27df-483f-a009-42303866394b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:10:00.698917174Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:12:23.691383966Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000a8e680\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-fncbll\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\\\",\\n  \\\"name\\\": \\\"1dcfdab9-27df-483f-a009-42303866394b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:10:00.698917174Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:12:23.691383966Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00094b230\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Length\": [\"503\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Request-Id\": [\n                            \"ee86427e-005d-4cc8-bd01-2b82a0056536\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADAEAST:20260318T221233Z:a23b9d02-f439-4d75-bcef-bec5f70b2688\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 2529055C24EC4D90ABA73CA7E8A5F278 Ref B: CHI261140504054 Ref C: 2026-03-18T22:12:33Z\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"11\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"a23b9d02-f439-4d75-bcef-bec5f70b2688\",\n                        ],\n                        \"Strict-Transport-Security\": [\n     ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000a8e6a0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-fncbll\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\",\n      \"name\": \"1dcfdab9-27df-483f-a009-42303866394b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:10:00.698917174Z\",\n      \"endTime\": \"2026-03-18T22:12:23.691383966Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-fncbll\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\\\",\\n  \\\"name\\\": \\\"1dcfdab9-27df-483f-a009-42303866394b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:10:00.698917174Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:12:23.691383966Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000a8e680\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-fncbll\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\\\",\\n  \\\"name\\\": \\\"1dcfdab9-27df-483f-a009-42303866394b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:10:00.698917174Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:12:23.691383966Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00094b230\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Length\": [\"503\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Request-Id\": [\n                            \"ee86427e-005d-4cc8-bd01-2b82a0056536\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADAEAST:20260318T221233Z:a23b9d02-f439-4d75-bcef-bec5f70b2688\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 2529055C24EC4D90ABA73CA7E8A5F278 Ref B: CHI261140504054 Ref C: 2026-03-18T22:12:33Z\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"11\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"a23b9d02-f439-4d75-bcef-bec5f70b2688\",\n                        ],\n                        \"Strict-Transport-Security\": [\n     ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000b943a0\u003e: \n    failed to create HCP cluster ea-list8n6lb4: failed waiting for cluster=\"ea-list8n6lb4\" in resourcegroup=\"ea-list-gwfwm5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\",\n      \"name\": \"b5aa74e8-43d5-4cff-b520-63b6b9163130\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T21:58:12.305820102Z\",\n      \"endTime\": \"2026-03-18T21:59:35.853803496Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-list8n6lb4: failed waiting for cluster=\\\"ea-list8n6lb4\\\" in resourcegroup=\\\"ea-list-gwfwm5\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\\\",\\n  \\\"name\\\": \\\"b5aa74e8-43d5-4cff-b520-63b6b9163130\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T21:58:12.305820102Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T21:59:35.853803496Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000b94380\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-list8n6lb4\\\" in resourcegroup=\\\"ea-list-gwfwm5\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\\\",\\n  \\\"name\\\": \\\"b5aa74e8-43d5-4cff-b520-63b6b9163130\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T21:58:12.305820102Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T21:59:35.853803496Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0011866c0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260318T215943Z:3c196c67-a4c1-40eb-abc4-c5d8f5d91317\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Date\": [\n                            \"Wed, 18 Mar 2026 21:59:43 GMT\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Ms-Request-Id\": [\n                            \"e2863777-8b30-4013-8910-cf48bfc93c57\",\n                        ],\n                        \"X-Envoy-Upstream-Service-Time\": [\"11\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 694EB5D5CF094E458C2CC1B85768A697 Ref B: CHI261140510060 Ref C: 2026-03-18T21:59:43Z\",\n                        ],\n                        \"Content-Length\": [\"503\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Expires...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000b943a0\u003e: \n    failed to create HCP cluster ea-list8n6lb4: failed waiting for cluster=\"ea-list8n6lb4\" in resourcegroup=\"ea-list-gwfwm5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\",\n      \"name\": \"b5aa74e8-43d5-4cff-b520-63b6b9163130\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T21:58:12.305820102Z\",\n      \"endTime\": \"2026-03-18T21:59:35.853803496Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-list8n6lb4: failed waiting for cluster=\\\"ea-list8n6lb4\\\" in resourcegroup=\\\"ea-list-gwfwm5\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\\\",\\n  \\\"name\\\": \\\"b5aa74e8-43d5-4cff-b520-63b6b9163130\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T21:58:12.305820102Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T21:59:35.853803496Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000b94380\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-list8n6lb4\\\" in resourcegroup=\\\"ea-list-gwfwm5\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\\\",\\n  \\\"name\\\": \\\"b5aa74e8-43d5-4cff-b520-63b6b9163130\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T21:58:12.305820102Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T21:59:35.853803496Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0011866c0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260318T215943Z:3c196c67-a4c1-40eb-abc4-c5d8f5d91317\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Date\": [\n                            \"Wed, 18 Mar 2026 21:59:43 GMT\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Ms-Request-Id\": [\n                            \"e2863777-8b30-4013-8910-cf48bfc93c57\",\n                        ],\n                        \"X-Envoy-Upstream-Service-Time\": [\"11\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 694EB5D5CF094E458C2CC1B85768A697 Ref B: CHI261140510060 Ref C: 2026-03-18T21:59:43Z\",\n                        ],\n                        \"Content-Length\": [\"503\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Expires...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"06635d020ca291cd1c2ba88618a73ef47ef44b12a92afe822f14d0c83ccde9f6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"175c269dca6d1137eef269c582abddb7ea6d16d67923c4a3c53772140d9877f0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"3b7a7bf63bccef1e745b4f8f6507bf4c5bcf0506c6ce750e6970677fb0f50399","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"72bbfe9d9170a9135f3c05006adad3408c43107f264ad7ae326669faadef2c9f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"79926993dbe90889442ba2b21af223fbbe8febc0cc9c8b33332060f1c383fcc5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"9ba23246eb42a095691a46b9cbd632f747017108f5eaa0f29a11cd9f1796fc29","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"d0717dc22e468ddd022ff528d4aecb3fdbb5c6dab0c4a12be5be6d1d52eb794c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"d15d41907b1259276c4ebb3dec5f8bc7b80cc8a5f9d4f453ea811cdba4d45fc6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"e6aef335aed58df551668c1c4fb6e2b1c9955eae709dc370d9435a1d3a5cdffd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"f15a96cf1199ee286ffca5a7a3a92b706c70d32e3672375967fe435f387de27b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"fa90fc4c3a8dc536971855f197279bd90253d75f6a03c9cea859289d5028f03e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-txz9bx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-txz9bx, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":7,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":7,"after_last_push_count":7,"trend_counts":[0,0,0,0,0,0,7],"trend_range":"2026-03-12..2026-03-18 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-02"],"prior_runs_affected":8,"prior_last_seen_at":"2026-03-07T02:00:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:96]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e6cb10\u003e: \n    failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-vm79bx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vm79bx\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0011648c0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000e6cae0\u003e{\n                msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vm79bx\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0011648c0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:96]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e6cb10\u003e: \n    failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-vm79bx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vm79bx\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0011648c0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000e6cae0\u003e{\n                msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vm79bx\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0011648c0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00101e2d0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-nxd528\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-nxd528\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001374460\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00101e2a0\u003e{\n                msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-nxd528\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001374460\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00101e2d0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-nxd528\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-nxd528\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001374460\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00101e2a0\u003e{\n                msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-nxd528\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001374460\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00142ccf0\u003e: \n    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-sjngpl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-sjngpl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00158dc10\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00142ccc0\u003e{\n                msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-sjngpl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc00158dc10\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00142ccf0\u003e: \n    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-sjngpl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-sjngpl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00158dc10\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00142ccc0\u003e{\n                msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-sjngpl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc00158dc10\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"0adb0e14ff90a50fd727e2352980ce875ced0b7130a59c2b7ed94a1d2accd443","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"20b718cbd2e3ab1df68073123c329c2df9c689fd937380268f7e88c5d24e952c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"29f0eae22dce4003fd469e952b9869eb3891a2ef3f974a873ee4dd1a59a06db0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"aea60a8b0ab44c615808be854a956d9a8caed07cbc9500e67544d4ee655aa8d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"b3d5ff4c6489e7b7eebdfc119d86db673a1197b69b52d19f8423004d3f858303","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"ba32603bd52a75ed7762808e5de2a5d3ad0e6b5a105d17eda82735596cad91f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"ebdd99780dc0ca89741307789c856554dc35c473e657c97cf89059a55b98274d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[0,0,0,0,1,0,1],"trend_range":"2026-03-12..2026-03-18 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-02"],"prior_runs_affected":5,"prior_last_seen_at":"2026-03-05T07:27:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"b8285d1fe5e0e06b0118077c74385230269c41bde5a189b9513acacc0d1a338e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":1,"after_last_push_count":17,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-12..2026-03-18 UTC","prior_windows_present":2,"prior_window_starts":["2026-02-23","2026-03-02"],"prior_runs_affected":11,"prior_last_seen_at":"2026-03-05T05:33:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["\u003e github.com/Azure/ARO-HCP/test/e2e.init.func15.2({0x7f54ec580e60, 0xc0009fb7d0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n                | \n                | By(\"waiting for first cluster to complete creation\")\n                \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n                | \tFrequency: framework.StandardPollInterval,\n                | })\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x827b940?, 0xc0009fb7d0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 84\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/18/26 23:20:55.083\n  \"ts\"=\"2026-03-18 23:20:55.083532\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-18 23:20:55.083598\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-kjjpjb\" \"resourceGroup\"=\"customer-rg-2hmq5s\"\n  \"ts\"=\"2026-03-18 23:20:55.333961\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-kjjpjb\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 23:20:56.369508\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-wlxsb8\" \"resourceGroup\"=\"customer-rg-2hmq5s\"\n  \"ts\"=\"2026-03-18 23:20:56.519635\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-wlxsb8\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 23:20:57.115820\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-18 23:20:57.115955\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-18 23:20:57.116022\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-18 23:20:57.117126\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-26\"\n  \"ts\"=\"2026-03-18 23:20:57.117146\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-110\"\n  \"ts\"=\"2026-03-18 23:20:57.117168\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-2hmq5s\"\n  \"ts\"=\"2026-03-18 23:20:57.117208\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-36\"\n  \"ts\"=\"2026-03-18 23:20:57.264846\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-110\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 23:20:57.327570\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-36\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 23:20:57.396893\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-110\" \"deployment\"=\"msi-deployment-110\"\n  \"ts\"=\"2026-03-18 23:20:57.432057\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-2hmq5s\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n  \"ts\"=\"2026-03-18 23:20:57.548409\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-36\" \"deployment\"=\"msi-deployment-36\"\n  \"ts\"=\"2026-03-18 23:20:57.629848\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-2hmq5s\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-wlxsb8\"\n  \"ts\"=\"2026-03-18 23:20:57.773291\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-26\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 23:20:57.819000\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-2hmq5s\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-03-18 23:20:57.904929\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-2hmq5s\" \"deployment\"=\"customer-infra-basic-hcp-cluster-kjjpjb\"\n  \"ts\"=\"2026-03-18 23:20:58.268158\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-26\" \"deployment\"=\"msi-deployment-26\"\n  \"ts\"=\"2026-03-18 23:20:58.435912\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-18 23:20:58.436063\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-18 23:20:58.436192\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-2hmq5s\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n\u003e github.com/Azure/ARO-HCP/test/e2e.init.func15.2({0x7f54ec580e60, 0xc0009fb7d0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n                | \n                | By(\"waiting for first cluster to complete creation\")\n                \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n                | \tFrequency: framework.StandardPollInterval,\n                | })\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x827b940?, 0xc0009fb7d0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 84\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/18/26 23:20:55.083\n  \"ts\"=\"2026-03-18 23:20:55.083532\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-18 23:20:55.083598\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-kjjpjb\" \"resourceGroup\"=\"customer-rg-2hmq5s\"\n  \"ts\"=\"2026-03-18 23:20:55.333961\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-kjjpjb\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 23:20:56.369508\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-wlxsb8\" \"resourceGroup\"=\"customer-rg-2hmq5s\"\n  \"ts\"=\"2026-03-18 23:20:56.519635\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-wlxsb8\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 23:20:57.115820\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-18 23:20:57.115955\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-18 23:20:57.116022\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-18 23:20:57.117126\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-26\"\n  \"ts\"=\"2026-03-18 23:20:57.117146\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-110\"\n  \"ts\"=\"2026-03-18 23:20:57.117168\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-2hmq5s\"\n  \"ts\"=\"2026-03-18 23:20:57.117208\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-36\"\n  \"ts\"=\"2026-03-18 23:20:57.264846\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-110\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 23:20:57.327570\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-36\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 23:20:57.396893\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-110\" \"deployment\"=\"msi-deployment-110\"\n  \"ts\"=\"2026-03-18 23:20:57.432057\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-2hmq5s\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n  \"ts\"=\"2026-03-18 23:20:57.548409\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-36\" \"deployment\"=\"msi-deployment-36\"\n  \"ts\"=\"2026-03-18 23:20:57.629848\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-2hmq5s\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-wlxsb8\"\n  \"ts\"=\"2026-03-18 23:20:57.773291\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-26\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 23:20:57.819000\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-2hmq5s\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-03-18 23:20:57.904929\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-2hmq5s\" \"deployment\"=\"customer-infra-basic-hcp-cluster-kjjpjb\"\n  \"ts\"=\"2026-03-18 23:20:58.268158\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-26\" \"deployment\"=\"msi-deployment-26\"\n  \"ts\"=\"2026-03-18 23:20:58.435912\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-18 23:20:58.436063\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-18 23:20:58.436192\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-2hmq5s\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"bae0429ac1980489e8f968319dc031c07dd05138b989654b348f26287e5c313e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ca7cc2da32e290afc8b2a3a18285c29643093d3821378dbf18bf57bc20cad012","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message ClientCertificateCredential: unable to resolve an endpoint: server response error: context deadline exceeded; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-12..2026-03-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001828480\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-99xb7j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\",\n      \"name\": \"6e36926c-e6c5-49f7-a607-9f56a4bbb73b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:05:01.130113225Z\",\n      \"endTime\": \"2026-03-18T22:06:29.307118365Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Failed to verify existence of managed resource group 'patch-name-99xb7j--managed': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"failed to get quota usage data: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-99xb7j\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\\",\\n  \\\"name\\\": \\\"6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:05:01.130113225Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:06:29.307118365Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to verify existence of managed resource group 'patch-name-99xb7j--managed': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"failed to get quota usage data: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc001828460\u003e{\n            msg: \"failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-99xb7j\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\\",\\n  \\\"name\\\": \\\"6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:05:01.130113225Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:06:29.307118365Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to verify existence of managed resource group 'patch-name-99xb7j--managed': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerErro...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001828480\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-99xb7j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\",\n      \"name\": \"6e36926c-e6c5-49f7-a607-9f56a4bbb73b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:05:01.130113225Z\",\n      \"endTime\": \"2026-03-18T22:06:29.307118365Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Failed to verify existence of managed resource group 'patch-name-99xb7j--managed': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"failed to get quota usage data: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-99xb7j\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\\",\\n  \\\"name\\\": \\\"6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:05:01.130113225Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:06:29.307118365Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to verify existence of managed resource group 'patch-name-99xb7j--managed': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"failed to get quota usage data: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc001828460\u003e{\n            msg: \"failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-99xb7j\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\\",\\n  \\\"name\\\": \\\"6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:05:01.130113225Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:06:29.307118365Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to verify existence of managed resource group 'patch-name-99xb7j--managed': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerErro...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"237ff6b17b8ec98ad232c04a0ed44127d05d48dc83399baccc02c34cebfbfc12","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ca156a6e6dc55851f861662371785d03e87c4fa68946334f3057fa599843ab7e","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":11.11111111111111,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-12..2026-03-18 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00133a3c0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-sv2zcp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\",\n      \"name\": \"b06360bf-0a54-4e7d-b9f1-3e5b6626253e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:17:00.823479898Z\",\n      \"endTime\": \"2026-03-18T22:18:09.152905722Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"insufficient public IP address quota: required 2, available 0\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Failed to get virtual network 'customer-vnet-name' details: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-120/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"failed to get network security group 'customer-nsg-name' details: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-sv2zcp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\\",\\n  \\\"name\\\": \\\"b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:17:00.823479898Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:18:09.152905722Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-name' details: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-120/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00133a3a0\u003e{\n            msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-sv2zcp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\\",\\n  \\\"name\\\": \\\"b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:17:00.823479898Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:18:09.152905722Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-name' details: Cl...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00133a3c0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-sv2zcp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\",\n      \"name\": \"b06360bf-0a54-4e7d-b9f1-3e5b6626253e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:17:00.823479898Z\",\n      \"endTime\": \"2026-03-18T22:18:09.152905722Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"insufficient public IP address quota: required 2, available 0\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Failed to get virtual network 'customer-vnet-name' details: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-120/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"failed to get network security group 'customer-nsg-name' details: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-sv2zcp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\\",\\n  \\\"name\\\": \\\"b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:17:00.823479898Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:18:09.152905722Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-name' details: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-120/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00133a3a0\u003e{\n            msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-sv2zcp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\\",\\n  \\\"name\\\": \\\"b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:17:00.823479898Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:18:09.152905722Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-name' details: Cl...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","signature_id":"fd62b1986336789b07747b1dea7d2f2f4649096a7807af24263707e890c6d458","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}