{"meta":{"date":"2026-03-18","timezone":"UTC","generated_at":"2026-08-30T14:39:44Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":9,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":1,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":7,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034418600969768960","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-18T23:56:06Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ebda8f3cdfe68578e3e3e3cfd112bcb4ae29fb40721bf0e18e43f614da6871f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034418600969768960","occurred_at":"2026-03-18T23:56:06Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-18T21:39:24Z"},"failed_at":["e2e"],"failed_test_count":22,"occurrences":[{"row_id":"0a06c2efe4685aba9fa68cf2bf3533cd8bcac1058e612ebe2d7a7a7ce09543bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","failed_at":"e2e","signature_id":"d0717dc22e468ddd022ff528d4aecb3fdbb5c6dab0c4a12be5be6d1d52eb794c","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0002e4c80\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-bnf7qs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a7266eb7-02fe-4e25-8a6b-80bf2b07af6c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a7266eb7-02fe-4e25-8a6b-80bf2b07af6c\",\n      \"name\": \"a7266eb7-02fe-4e25-8a6b-80bf2b07af6c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:04:06.780626426Z\",\n      \"endTime\": \"2026-03-18T22:04:50.639636342Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-bnf7qs\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a7266eb7-02fe-4e25-8a6b-80bf2b07af6c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a7266eb7-02fe-4e25-8a6b-80bf2b07af6c\\\",\\n  \\\"name\\\": \\\"a7266eb7-02fe-4e25-8a6b-80bf2b07af6c\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:04:06.780626426Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:04:50.639636342Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0002e4c60\u003e{\n            msg: \"failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-bnf7qs\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a7266eb7-02fe-4e25-8a6b-80bf2b07af6c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a7266eb7-02fe-4e25-8a6b-80bf2b07af6c\\\",\\n  \\\"name\\\": \\\"a7266eb7-02fe-4e25-8a6b-80bf2b07af6c\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:04:06.780626426Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:04:50.639636342Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc001556900\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Request-Id\": [\n                            \"68337901-b202-4387-8b62-a5d5fa9a8a87\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/86f586e0-8021-49ae-8dab-791e7e4ddc69\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"b2f55a6e-086e-4aea-93f6-026a3332b1ba\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 340E61C37B254A58B9B02493846AE154 Ref B: CHI261140503054 Ref C: 2026-03-18T22:04:57Z\",\n                        ],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Ratelimit-Rem...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0002e4c80\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-bnf7qs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a7266eb7-02fe-4e25-8a6b-80bf2b07af6c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a7266eb7-02fe-4e25-8a6b-80bf2b07af6c\",\n      \"name\": \"a7266eb7-02fe-4e25-8a6b-80bf2b07af6c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:04:06.780626426Z\",\n      \"endTime\": \"2026-03-18T22:04:50.639636342Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-bnf7qs\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a7266eb7-02fe-4e25-8a6b-80bf2b07af6c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a7266eb7-02fe-4e25-8a6b-80bf2b07af6c\\\",\\n  \\\"name\\\": \\\"a7266eb7-02fe-4e25-8a6b-80bf2b07af6c\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:04:06.780626426Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:04:50.639636342Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0002e4c60\u003e{\n            msg: \"failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-bnf7qs\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a7266eb7-02fe-4e25-8a6b-80bf2b07af6c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a7266eb7-02fe-4e25-8a6b-80bf2b07af6c\\\",\\n  \\\"name\\\": \\\"a7266eb7-02fe-4e25-8a6b-80bf2b07af6c\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:04:06.780626426Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:04:50.639636342Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc001556900\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Request-Id\": [\n                            \"68337901-b202-4387-8b62-a5d5fa9a8a87\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/86f586e0-8021-49ae-8dab-791e7e4ddc69\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"b2f55a6e-086e-4aea-93f6-026a3332b1ba\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 340E61C37B254A58B9B02493846AE154 Ref B: CHI261140503054 Ref C: 2026-03-18T22:04:57Z\",\n                        ],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Ratelimit-Rem...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8a0eb43d82c81f5ac221f93e261d7d3906eccf9364890faef15f25844bdadb79","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0b62a22bd44547d23df3224a9351bf890acdcc3eaf19fa2168ca816ccc85686f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","failed_at":"e2e","signature_id":"20b718cbd2e3ab1df68073123c329c2df9c689fd937380268f7e88c5d24e952c","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00101e2d0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-nxd528\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-nxd528\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001374460\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00101e2a0\u003e{\n                msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-nxd528\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001374460\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00101e2d0\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-nxd528\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-nxd528\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001374460\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00101e2a0\u003e{\n                msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-nxd528\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001374460\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-nxd528\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-txz9bx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-txz9bx, error: context deadline exceeded"}},{"row_id":"3ff6ca2d23ab7bc687692bbf32f6a86fcc2b9eb7a05e58855df111bbf51c2e50","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","failed_at":"e2e","signature_id":"0adb0e14ff90a50fd727e2352980ce875ced0b7130a59c2b7ed94a1d2accd443","test_name":"Customer should be able to create an HCP cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:96]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e6cb10\u003e: \n    failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-vm79bx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vm79bx\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0011648c0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000e6cae0\u003e{\n                msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vm79bx\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0011648c0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:96]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e6cb10\u003e: \n    failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-vm79bx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster basic-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vm79bx\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0011648c0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000e6cae0\u003e{\n                msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vm79bx\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0011648c0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-hcp-cluster in resource group basic-cluster-vm79bx\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-txz9bx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-txz9bx, error: context deadline exceeded"}},{"row_id":"403df8b3468a86386a9a8ee75720896d21bd12af1584648c07be7882fe91f0e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","failed_at":"e2e","signature_id":"b3d5ff4c6489e7b7eebdfc119d86db673a1197b69b52d19f8423004d3f858303","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000fc8e70\u003e: \n    failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-zdx4fc, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-zdx4fc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-zdx4fc, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-zdx4fc, error: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-zdx4fc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-zdx4fc, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0011f5360\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-zdx4fc\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000fc8e40\u003e{\n                msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-zdx4fc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-zdx4fc, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0011f5360\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-zdx4fc\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000fc8e70\u003e: \n    failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-zdx4fc, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-zdx4fc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-zdx4fc, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-zdx4fc, error: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-zdx4fc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-zdx4fc, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0011f5360\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-zdx4fc\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000fc8e40\u003e{\n                msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-zdx4fc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-zdx4fc, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0011f5360\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-zdx4fc\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-txz9bx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-txz9bx, error: context deadline exceeded"}},{"row_id":"41a6fe26a947515504e8881e7eb0d34f0277ba1bb911fb5419fac711426ca7c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","failed_at":"e2e","signature_id":"bae0429ac1980489e8f968319dc031c07dd05138b989654b348f26287e5c313e","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"\u003e github.com/Azure/ARO-HCP/test/e2e.init.func15.2({0x7f54ec580e60, 0xc0009fb7d0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n                | \n                | By(\"waiting for first cluster to complete creation\")\n                \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n                | \tFrequency: framework.StandardPollInterval,\n                | })\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x827b940?, 0xc0009fb7d0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 84\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/18/26 23:20:55.083\n  \"ts\"=\"2026-03-18 23:20:55.083532\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-18 23:20:55.083598\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-kjjpjb\" \"resourceGroup\"=\"customer-rg-2hmq5s\"\n  \"ts\"=\"2026-03-18 23:20:55.333961\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-kjjpjb\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 23:20:56.369508\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-wlxsb8\" \"resourceGroup\"=\"customer-rg-2hmq5s\"\n  \"ts\"=\"2026-03-18 23:20:56.519635\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-wlxsb8\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 23:20:57.115820\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-18 23:20:57.115955\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-18 23:20:57.116022\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-18 23:20:57.117126\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-26\"\n  \"ts\"=\"2026-03-18 23:20:57.117146\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-110\"\n  \"ts\"=\"2026-03-18 23:20:57.117168\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-2hmq5s\"\n  \"ts\"=\"2026-03-18 23:20:57.117208\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-36\"\n  \"ts\"=\"2026-03-18 23:20:57.264846\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-110\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 23:20:57.327570\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-36\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 23:20:57.396893\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-110\" \"deployment\"=\"msi-deployment-110\"\n  \"ts\"=\"2026-03-18 23:20:57.432057\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-2hmq5s\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n  \"ts\"=\"2026-03-18 23:20:57.548409\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-36\" \"deployment\"=\"msi-deployment-36\"\n  \"ts\"=\"2026-03-18 23:20:57.629848\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-2hmq5s\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-wlxsb8\"\n  \"ts\"=\"2026-03-18 23:20:57.773291\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-26\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 23:20:57.819000\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-2hmq5s\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-03-18 23:20:57.904929\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-2hmq5s\" \"deployment\"=\"customer-infra-basic-hcp-cluster-kjjpjb\"\n  \"ts\"=\"2026-03-18 23:20:58.268158\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-26\" \"deployment\"=\"msi-deployment-26\"\n  \"ts\"=\"2026-03-18 23:20:58.435912\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-18 23:20:58.436063\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-18 23:20:58.436192\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-2hmq5s\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n\u003e github.com/Azure/ARO-HCP/test/e2e.init.func15.2({0x7f54ec580e60, 0xc0009fb7d0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n                | \n                | By(\"waiting for first cluster to complete creation\")\n                \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n                | \tFrequency: framework.StandardPollInterval,\n                | })\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x827b940?, 0xc0009fb7d0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 84\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/18/26 23:20:55.083\n  \"ts\"=\"2026-03-18 23:20:55.083532\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-18 23:20:55.083598\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-kjjpjb\" \"resourceGroup\"=\"customer-rg-2hmq5s\"\n  \"ts\"=\"2026-03-18 23:20:55.333961\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-kjjpjb\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 23:20:56.369508\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-wlxsb8\" \"resourceGroup\"=\"customer-rg-2hmq5s\"\n  \"ts\"=\"2026-03-18 23:20:56.519635\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-wlxsb8\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-18 23:20:57.115820\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-18 23:20:57.115955\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-18 23:20:57.116022\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-18 23:20:57.117126\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-26\"\n  \"ts\"=\"2026-03-18 23:20:57.117146\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-110\"\n  \"ts\"=\"2026-03-18 23:20:57.117168\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-2hmq5s\"\n  \"ts\"=\"2026-03-18 23:20:57.117208\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-36\"\n  \"ts\"=\"2026-03-18 23:20:57.264846\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-110\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 23:20:57.327570\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-36\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 23:20:57.396893\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-110\" \"deployment\"=\"msi-deployment-110\"\n  \"ts\"=\"2026-03-18 23:20:57.432057\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-2hmq5s\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n  \"ts\"=\"2026-03-18 23:20:57.548409\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-36\" \"deployment\"=\"msi-deployment-36\"\n  \"ts\"=\"2026-03-18 23:20:57.629848\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-2hmq5s\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-wlxsb8\"\n  \"ts\"=\"2026-03-18 23:20:57.773291\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-26\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-18 23:20:57.819000\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-2hmq5s\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-03-18 23:20:57.904929\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-2hmq5s\" \"deployment\"=\"customer-infra-basic-hcp-cluster-kjjpjb\"\n  \"ts\"=\"2026-03-18 23:20:58.268158\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-26\" \"deployment\"=\"msi-deployment-26\"\n  \"ts\"=\"2026-03-18 23:20:58.435912\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-18 23:20:58.436063\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-18 23:20:58.436192\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-2hmq5s\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"529cd71ae45be53acdc306f8b3bded19eb979dc65223d299d9ba6dcb3821a0c3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","failed_at":"e2e","signature_id":"aea60a8b0ab44c615808be854a956d9a8caed07cbc9500e67544d4ee655aa8d6","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014de840\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-txz9bx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-txz9bx, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-txz9bx\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-txz9bx, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000c651e0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-txz9bx\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014de840\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-txz9bx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-txz9bx, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-txz9bx\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-txz9bx, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000c651e0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-txz9bx\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-txz9bx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-txz9bx, error: context deadline exceeded"}},{"row_id":"57bafca071c535b4bf7d7bb86f0b2ec2794138a81565a936f8afa0fdceea4b4c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","failed_at":"e2e","signature_id":"fa90fc4c3a8dc536971855f197279bd90253d75f6a03c9cea859289d5028f03e","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009001a0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-94f5n8: failed waiting for cluster=\"cluster-zstream-4-21-94f5n8\" in resourcegroup=\"rg-zstream-upgrade-4-21-64j8h4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cac1b2e-d936-402c-86be-44b7a37d4ae1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cac1b2e-d936-402c-86be-44b7a37d4ae1\",\n      \"name\": \"8cac1b2e-d936-402c-86be-44b7a37d4ae1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:09:58.306382688Z\",\n      \"endTime\": \"2026-03-18T22:11:58.452014263Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-21-94f5n8: failed waiting for cluster=\\\"cluster-zstream-4-21-94f5n8\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-64j8h4\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cac1b2e-d936-402c-86be-44b7a37d4ae1\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cac1b2e-d936-402c-86be-44b7a37d4ae1\\\",\\n  \\\"name\\\": \\\"8cac1b2e-d936-402c-86be-44b7a37d4ae1\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:09:58.306382688Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:11:58.452014263Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000900180\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-94f5n8\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-64j8h4\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cac1b2e-d936-402c-86be-44b7a37d4ae1\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cac1b2e-d936-402c-86be-44b7a37d4ae1\\\",\\n  \\\"name\\\": \\\"8cac1b2e-d936-402c-86be-44b7a37d4ae1\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:09:58.306382688Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:11:58.452014263Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000575020\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260318T221200Z:cc127123-120b-4dba-b721-6d7a1221b757\",\n                        ],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Request-Id\": [\n                            \"88b27e98-71ad-423d-941a-3dfed915fe8e\",\n                        ],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"cc127123-120b-4dba-b721-6d7a1221b757\",\n                        ],\n                        \"Date\": [\n                            \"Wed, 18 Mar 2026 22:12:00 GMT\",\n                        ],\n                        \"Content-Length\": [\"503\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/f9182697-47ef-450e-985b-d1bab32508ff\",\n                        ],\n      ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009001a0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-94f5n8: failed waiting for cluster=\"cluster-zstream-4-21-94f5n8\" in resourcegroup=\"rg-zstream-upgrade-4-21-64j8h4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cac1b2e-d936-402c-86be-44b7a37d4ae1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cac1b2e-d936-402c-86be-44b7a37d4ae1\",\n      \"name\": \"8cac1b2e-d936-402c-86be-44b7a37d4ae1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:09:58.306382688Z\",\n      \"endTime\": \"2026-03-18T22:11:58.452014263Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-21-94f5n8: failed waiting for cluster=\\\"cluster-zstream-4-21-94f5n8\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-64j8h4\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cac1b2e-d936-402c-86be-44b7a37d4ae1\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cac1b2e-d936-402c-86be-44b7a37d4ae1\\\",\\n  \\\"name\\\": \\\"8cac1b2e-d936-402c-86be-44b7a37d4ae1\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:09:58.306382688Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:11:58.452014263Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000900180\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-94f5n8\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-64j8h4\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cac1b2e-d936-402c-86be-44b7a37d4ae1\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8cac1b2e-d936-402c-86be-44b7a37d4ae1\\\",\\n  \\\"name\\\": \\\"8cac1b2e-d936-402c-86be-44b7a37d4ae1\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:09:58.306382688Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:11:58.452014263Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000575020\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260318T221200Z:cc127123-120b-4dba-b721-6d7a1221b757\",\n                        ],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Request-Id\": [\n                            \"88b27e98-71ad-423d-941a-3dfed915fe8e\",\n                        ],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"cc127123-120b-4dba-b721-6d7a1221b757\",\n                        ],\n                        \"Date\": [\n                            \"Wed, 18 Mar 2026 22:12:00 GMT\",\n                        ],\n                        \"Content-Length\": [\"503\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/f9182697-47ef-450e-985b-d1bab32508ff\",\n                        ],\n      ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8a0eb43d82c81f5ac221f93e261d7d3906eccf9364890faef15f25844bdadb79","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"587b01f73e08b51bb1424f195f58193bd06e403a1775d3b1f0be08cb1f915e5c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","failed_at":"e2e","signature_id":"ebdd99780dc0ca89741307789c856554dc35c473e657c97cf89059a55b98274d","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000de6db0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-fjpggg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-fjpggg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-fjpggg, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-fjpggg, error: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-fjpggg\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-fjpggg, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001575960\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-fjpggg\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000de6d80\u003e{\n                msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-fjpggg\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-fjpggg, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001575960\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-fjpggg\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000de6db0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-fjpggg, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-fjpggg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-fjpggg, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-fjpggg, error: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-fjpggg\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-fjpggg, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001575960\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-fjpggg\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000de6d80\u003e{\n                msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-fjpggg\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-fjpggg, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001575960\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-fjpggg\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-txz9bx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-txz9bx, error: context deadline exceeded"}},{"row_id":"726445930733bf79be77a2c3e61341ecbc8048757df180d724224395e76df8e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","failed_at":"e2e","signature_id":"79926993dbe90889442ba2b21af223fbbe8febc0cc9c8b33332060f1c383fcc5","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000a02180\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-dngc6m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e959e2ef-e55d-46f1-8f37-18a399818306\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e959e2ef-e55d-46f1-8f37-18a399818306\",\n      \"name\": \"e959e2ef-e55d-46f1-8f37-18a399818306\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T21:58:25.256129849Z\",\n      \"endTime\": \"2026-03-18T21:59:35.785955056Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-dngc6m\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e959e2ef-e55d-46f1-8f37-18a399818306\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e959e2ef-e55d-46f1-8f37-18a399818306\\\",\\n  \\\"name\\\": \\\"e959e2ef-e55d-46f1-8f37-18a399818306\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T21:58:25.256129849Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T21:59:35.785955056Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000a02160\u003e{\n            msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-dngc6m\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e959e2ef-e55d-46f1-8f37-18a399818306\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e959e2ef-e55d-46f1-8f37-18a399818306\\\",\\n  \\\"name\\\": \\\"e959e2ef-e55d-46f1-8f37-18a399818306\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T21:58:25.256129849Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T21:59:35.785955056Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000978450\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Envoy-Upstream-Service-Time\": [\"9\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"6b4b037b-5eae-4b4a-89a0-993dc36429cf\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Date\": [\n                            \"Wed, 18 Mar 2026 21:59:36 GMT\",\n                        ],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadaeast/7240e2d5-207e-4f6d-805f-097d383fb77f\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADAEAST:20260318T215936Z:6b4b037b-5eae-4b4a-89a0-993dc36429cf\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"Cache-Control\": [\"no-...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000a02180\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-dngc6m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e959e2ef-e55d-46f1-8f37-18a399818306\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e959e2ef-e55d-46f1-8f37-18a399818306\",\n      \"name\": \"e959e2ef-e55d-46f1-8f37-18a399818306\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T21:58:25.256129849Z\",\n      \"endTime\": \"2026-03-18T21:59:35.785955056Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-dngc6m\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e959e2ef-e55d-46f1-8f37-18a399818306\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e959e2ef-e55d-46f1-8f37-18a399818306\\\",\\n  \\\"name\\\": \\\"e959e2ef-e55d-46f1-8f37-18a399818306\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T21:58:25.256129849Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T21:59:35.785955056Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000a02160\u003e{\n            msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-dngc6m\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e959e2ef-e55d-46f1-8f37-18a399818306\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e959e2ef-e55d-46f1-8f37-18a399818306\\\",\\n  \\\"name\\\": \\\"e959e2ef-e55d-46f1-8f37-18a399818306\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T21:58:25.256129849Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T21:59:35.785955056Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000978450\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Envoy-Upstream-Service-Time\": [\"9\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"6b4b037b-5eae-4b4a-89a0-993dc36429cf\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Date\": [\n                            \"Wed, 18 Mar 2026 21:59:36 GMT\",\n                        ],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadaeast/7240e2d5-207e-4f6d-805f-097d383fb77f\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADAEAST:20260318T215936Z:6b4b037b-5eae-4b4a-89a0-993dc36429cf\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"Cache-Control\": [\"no-...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8a0eb43d82c81f5ac221f93e261d7d3906eccf9364890faef15f25844bdadb79","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7b5383e5ea7443e3a50430b5a01d306b6946c74e6400cf7c8dc0fccd74b8ad96","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","failed_at":"e2e","signature_id":"d15d41907b1259276c4ebb3dec5f8bc7b80cc8a5f9d4f453ea811cdba4d45fc6","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009927a0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2x5tqd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/15b94b31-6297-4e0a-818d-5ea9d03e264c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/15b94b31-6297-4e0a-818d-5ea9d03e264c\",\n      \"name\": \"15b94b31-6297-4e0a-818d-5ea9d03e264c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T21:58:09.542609231Z\",\n      \"endTime\": \"2026-03-18T21:59:35.888371839Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-2x5tqd\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/15b94b31-6297-4e0a-818d-5ea9d03e264c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/15b94b31-6297-4e0a-818d-5ea9d03e264c\\\",\\n  \\\"name\\\": \\\"15b94b31-6297-4e0a-818d-5ea9d03e264c\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T21:58:09.542609231Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T21:59:35.888371839Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000992780\u003e{\n            msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-2x5tqd\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/15b94b31-6297-4e0a-818d-5ea9d03e264c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/15b94b31-6297-4e0a-818d-5ea9d03e264c\\\",\\n  \\\"name\\\": \\\"15b94b31-6297-4e0a-818d-5ea9d03e264c\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T21:58:09.542609231Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T21:59:35.888371839Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000928b40\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"503\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"85a78c11-e075-4480-8bc1-622a8ec78799\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 5C0E582900C8470A8392AAB9D62C27B0 Ref B: CHI261140504040 Ref C: 2026-03-18T21:59:41Z\",\n                        ],\n                        \"Date\": [\n                            \"Wed, 18 Mar 2026 21:59:40 GMT\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3748\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Request-Id\": [\n                            \"ee2f9f9d-ea91-4be1-8440-27274248be3e\",\n                        ],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009927a0\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-2x5tqd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/15b94b31-6297-4e0a-818d-5ea9d03e264c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/15b94b31-6297-4e0a-818d-5ea9d03e264c\",\n      \"name\": \"15b94b31-6297-4e0a-818d-5ea9d03e264c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T21:58:09.542609231Z\",\n      \"endTime\": \"2026-03-18T21:59:35.888371839Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-2x5tqd\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/15b94b31-6297-4e0a-818d-5ea9d03e264c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/15b94b31-6297-4e0a-818d-5ea9d03e264c\\\",\\n  \\\"name\\\": \\\"15b94b31-6297-4e0a-818d-5ea9d03e264c\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T21:58:09.542609231Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T21:59:35.888371839Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000992780\u003e{\n            msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-2x5tqd\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/15b94b31-6297-4e0a-818d-5ea9d03e264c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/15b94b31-6297-4e0a-818d-5ea9d03e264c\\\",\\n  \\\"name\\\": \\\"15b94b31-6297-4e0a-818d-5ea9d03e264c\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T21:58:09.542609231Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T21:59:35.888371839Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000928b40\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"503\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"85a78c11-e075-4480-8bc1-622a8ec78799\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 5C0E582900C8470A8392AAB9D62C27B0 Ref B: CHI261140504040 Ref C: 2026-03-18T21:59:41Z\",\n                        ],\n                        \"Date\": [\n                            \"Wed, 18 Mar 2026 21:59:40 GMT\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3748\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Request-Id\": [\n                            \"ee2f9f9d-ea91-4be1-8440-27274248be3e\",\n                        ],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8a0eb43d82c81f5ac221f93e261d7d3906eccf9364890faef15f25844bdadb79","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9394c2ee121a884319e8ba3e67cced0e3be9380f2b4a801c38c3e8f381b4069a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","failed_at":"e2e","signature_id":"29f0eae22dce4003fd469e952b9869eb3891a2ef3f974a873ee4dd1a59a06db0","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00142ccf0\u003e: \n    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-sjngpl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-sjngpl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00158dc10\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00142ccc0\u003e{\n                msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-sjngpl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc00158dc10\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00142ccf0\u003e: \n    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-sjngpl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-sjngpl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00158dc10\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00142ccc0\u003e{\n                msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-sjngpl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc00158dc10\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-sjngpl\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-txz9bx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-txz9bx, error: context deadline exceeded"}},{"row_id":"9de15237799a14cc9a7f02236a015cf8b2ed0e44767bf7774c677f4c9279fc0e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","failed_at":"e2e","signature_id":"fd62b1986336789b07747b1dea7d2f2f4649096a7807af24263707e890c6d458","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00133a3c0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-sv2zcp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\",\n      \"name\": \"b06360bf-0a54-4e7d-b9f1-3e5b6626253e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:17:00.823479898Z\",\n      \"endTime\": \"2026-03-18T22:18:09.152905722Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"insufficient public IP address quota: required 2, available 0\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Failed to get virtual network 'customer-vnet-name' details: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-120/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"failed to get network security group 'customer-nsg-name' details: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-sv2zcp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\\",\\n  \\\"name\\\": \\\"b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:17:00.823479898Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:18:09.152905722Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-name' details: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-120/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00133a3a0\u003e{\n            msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-sv2zcp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\\",\\n  \\\"name\\\": \\\"b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:17:00.823479898Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:18:09.152905722Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-name' details: Cl...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00133a3c0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-sv2zcp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\",\n      \"name\": \"b06360bf-0a54-4e7d-b9f1-3e5b6626253e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:17:00.823479898Z\",\n      \"endTime\": \"2026-03-18T22:18:09.152905722Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"insufficient public IP address quota: required 2, available 0\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Failed to get virtual network 'customer-vnet-name' details: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-120/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"failed to get network security group 'customer-nsg-name' details: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-sv2zcp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\\",\\n  \\\"name\\\": \\\"b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:17:00.823479898Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:18:09.152905722Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-name' details: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"failed to get managed identity '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-test-msi-containers-prod-120/providers/Microsoft.ManagedIdentity/userAssignedIdentities/ingress': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"failed to get network security group 'customer-nsg-name' details: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00133a3a0\u003e{\n            msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-sv2zcp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\\",\\n  \\\"name\\\": \\\"b06360bf-0a54-4e7d-b9f1-3e5b6626253e\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:17:00.823479898Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:18:09.152905722Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-name' details: Cl...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ca156a6e6dc55851f861662371785d03e87c4fa68946334f3057fa599843ab7e","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"a778c9be7cef41b0d4e412882ae0886e64bc5d0920d4ed66c524c3f5a1fa4eb3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","failed_at":"e2e","signature_id":"ba32603bd52a75ed7762808e5de2a5d3ad0e6b5a105d17eda82735596cad91f1","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010cc570\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-jrpfvc, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-jrpfvc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-jrpfvc, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-jrpfvc, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-jrpfvc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-jrpfvc, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000fde070\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-jrpfvc\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0010cc540\u003e{\n                msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-jrpfvc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-jrpfvc, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000fde070\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-jrpfvc\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010cc570\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-jrpfvc, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-jrpfvc\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-jrpfvc, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-jrpfvc, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-jrpfvc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-jrpfvc, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000fde070\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-jrpfvc\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0010cc540\u003e{\n                msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-jrpfvc\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-jrpfvc, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000fde070\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-jrpfvc\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-txz9bx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-txz9bx, error: context deadline exceeded"}},{"row_id":"b5d62659ee619b582a8c38c59fe44db991d57a7d350fd333500845440fe34973","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","failed_at":"e2e","signature_id":"175c269dca6d1137eef269c582abddb7ea6d16d67923c4a3c53772140d9877f0","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000a8e6a0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-fncbll\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\",\n      \"name\": \"1dcfdab9-27df-483f-a009-42303866394b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:10:00.698917174Z\",\n      \"endTime\": \"2026-03-18T22:12:23.691383966Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-fncbll\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\\\",\\n  \\\"name\\\": \\\"1dcfdab9-27df-483f-a009-42303866394b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:10:00.698917174Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:12:23.691383966Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000a8e680\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-fncbll\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\\\",\\n  \\\"name\\\": \\\"1dcfdab9-27df-483f-a009-42303866394b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:10:00.698917174Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:12:23.691383966Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00094b230\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Length\": [\"503\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Request-Id\": [\n                            \"ee86427e-005d-4cc8-bd01-2b82a0056536\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADAEAST:20260318T221233Z:a23b9d02-f439-4d75-bcef-bec5f70b2688\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 2529055C24EC4D90ABA73CA7E8A5F278 Ref B: CHI261140504054 Ref C: 2026-03-18T22:12:33Z\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"11\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"a23b9d02-f439-4d75-bcef-bec5f70b2688\",\n                        ],\n                        \"Strict-Transport-Security\": [\n     ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000a8e6a0\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-fncbll\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\",\n      \"name\": \"1dcfdab9-27df-483f-a009-42303866394b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:10:00.698917174Z\",\n      \"endTime\": \"2026-03-18T22:12:23.691383966Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-cluster: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-fncbll\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\\\",\\n  \\\"name\\\": \\\"1dcfdab9-27df-483f-a009-42303866394b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:10:00.698917174Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:12:23.691383966Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000a8e680\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-fncbll\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1dcfdab9-27df-483f-a009-42303866394b\\\",\\n  \\\"name\\\": \\\"1dcfdab9-27df-483f-a009-42303866394b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:10:00.698917174Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:12:23.691383966Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00094b230\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Length\": [\"503\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Request-Id\": [\n                            \"ee86427e-005d-4cc8-bd01-2b82a0056536\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADAEAST:20260318T221233Z:a23b9d02-f439-4d75-bcef-bec5f70b2688\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 2529055C24EC4D90ABA73CA7E8A5F278 Ref B: CHI261140504054 Ref C: 2026-03-18T22:12:33Z\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"11\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"a23b9d02-f439-4d75-bcef-bec5f70b2688\",\n                        ],\n                        \"Strict-Transport-Security\": [\n     ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8a0eb43d82c81f5ac221f93e261d7d3906eccf9364890faef15f25844bdadb79","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c03ade92aef8c3ed5f27c08a03b01ce61ce07ef17ab32ea587d0c15b969c0255","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","failed_at":"e2e","signature_id":"72bbfe9d9170a9135f3c05006adad3408c43107f264ad7ae326669faadef2c9f","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001368740\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-5g465c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/90443756-21e0-463a-9d79-af79062353c8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/90443756-21e0-463a-9d79-af79062353c8\",\n      \"name\": \"90443756-21e0-463a-9d79-af79062353c8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:03:11.273093527Z\",\n      \"endTime\": \"2026-03-18T22:04:50.652269195Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-5g465c\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/90443756-21e0-463a-9d79-af79062353c8\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/90443756-21e0-463a-9d79-af79062353c8\\\",\\n  \\\"name\\\": \\\"90443756-21e0-463a-9d79-af79062353c8\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:03:11.273093527Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:04:50.652269195Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc001368720\u003e{\n            msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-5g465c\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/90443756-21e0-463a-9d79-af79062353c8\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/90443756-21e0-463a-9d79-af79062353c8\\\",\\n  \\\"name\\\": \\\"90443756-21e0-463a-9d79-af79062353c8\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:03:11.273093527Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:04:50.652269195Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc001024b10\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Type\": [\"application/json\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Request-Id\": [\n                            \"322ecb3a-cca7-41b3-b384-68f9e35de431\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3748\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADAEAST:20260318T220453Z:8c67fafb-e096-4b92-8c0f-a92b31e7fba0\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: F9AD2BF3996148F2860FAE04B9F244ED Ref B: CHI261140504023 Ref C: 2026-03-18T22:04:53Z\",\n                        ],\n                        \"Content-Length\": [\"503\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"11\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"248\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"8c67fafb-e096-4b92-8c0f-a92b31e7fba0\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Date\": [\n...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001368740\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-5g465c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/90443756-21e0-463a-9d79-af79062353c8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/90443756-21e0-463a-9d79-af79062353c8\",\n      \"name\": \"90443756-21e0-463a-9d79-af79062353c8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:03:11.273093527Z\",\n      \"endTime\": \"2026-03-18T22:04:50.652269195Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-5g465c\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/90443756-21e0-463a-9d79-af79062353c8\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/90443756-21e0-463a-9d79-af79062353c8\\\",\\n  \\\"name\\\": \\\"90443756-21e0-463a-9d79-af79062353c8\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:03:11.273093527Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:04:50.652269195Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc001368720\u003e{\n            msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-5g465c\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/90443756-21e0-463a-9d79-af79062353c8\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/90443756-21e0-463a-9d79-af79062353c8\\\",\\n  \\\"name\\\": \\\"90443756-21e0-463a-9d79-af79062353c8\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:03:11.273093527Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:04:50.652269195Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc001024b10\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Type\": [\"application/json\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Request-Id\": [\n                            \"322ecb3a-cca7-41b3-b384-68f9e35de431\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3748\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADAEAST:20260318T220453Z:8c67fafb-e096-4b92-8c0f-a92b31e7fba0\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: F9AD2BF3996148F2860FAE04B9F244ED Ref B: CHI261140504023 Ref C: 2026-03-18T22:04:53Z\",\n                        ],\n                        \"Content-Length\": [\"503\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"11\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"248\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"8c67fafb-e096-4b92-8c0f-a92b31e7fba0\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Date\": [\n...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8a0eb43d82c81f5ac221f93e261d7d3906eccf9364890faef15f25844bdadb79","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c079068f4b026106e4239bd132c78dd8cd748127b4910c798cbfd7be628d0eb7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","failed_at":"e2e","signature_id":"b8285d1fe5e0e06b0118077c74385230269c41bde5a189b9513acacc0d1a338e","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes"}},{"row_id":"c4c980c7655fe7d686776686110f986dbe0ce79842777aa9b149c9fcd4fbeff5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","failed_at":"e2e","signature_id":"3b7a7bf63bccef1e745b4f8f6507bf4c5bcf0506c6ce750e6970677fb0f50399","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000b943a0\u003e: \n    failed to create HCP cluster ea-list8n6lb4: failed waiting for cluster=\"ea-list8n6lb4\" in resourcegroup=\"ea-list-gwfwm5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\",\n      \"name\": \"b5aa74e8-43d5-4cff-b520-63b6b9163130\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T21:58:12.305820102Z\",\n      \"endTime\": \"2026-03-18T21:59:35.853803496Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-list8n6lb4: failed waiting for cluster=\\\"ea-list8n6lb4\\\" in resourcegroup=\\\"ea-list-gwfwm5\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\\\",\\n  \\\"name\\\": \\\"b5aa74e8-43d5-4cff-b520-63b6b9163130\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T21:58:12.305820102Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T21:59:35.853803496Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000b94380\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-list8n6lb4\\\" in resourcegroup=\\\"ea-list-gwfwm5\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\\\",\\n  \\\"name\\\": \\\"b5aa74e8-43d5-4cff-b520-63b6b9163130\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T21:58:12.305820102Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T21:59:35.853803496Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0011866c0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260318T215943Z:3c196c67-a4c1-40eb-abc4-c5d8f5d91317\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Date\": [\n                            \"Wed, 18 Mar 2026 21:59:43 GMT\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Ms-Request-Id\": [\n                            \"e2863777-8b30-4013-8910-cf48bfc93c57\",\n                        ],\n                        \"X-Envoy-Upstream-Service-Time\": [\"11\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 694EB5D5CF094E458C2CC1B85768A697 Ref B: CHI261140510060 Ref C: 2026-03-18T21:59:43Z\",\n                        ],\n                        \"Content-Length\": [\"503\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Expires...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000b943a0\u003e: \n    failed to create HCP cluster ea-list8n6lb4: failed waiting for cluster=\"ea-list8n6lb4\" in resourcegroup=\"ea-list-gwfwm5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\",\n      \"name\": \"b5aa74e8-43d5-4cff-b520-63b6b9163130\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T21:58:12.305820102Z\",\n      \"endTime\": \"2026-03-18T21:59:35.853803496Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-list8n6lb4: failed waiting for cluster=\\\"ea-list8n6lb4\\\" in resourcegroup=\\\"ea-list-gwfwm5\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\\\",\\n  \\\"name\\\": \\\"b5aa74e8-43d5-4cff-b520-63b6b9163130\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T21:58:12.305820102Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T21:59:35.853803496Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000b94380\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-list8n6lb4\\\" in resourcegroup=\\\"ea-list-gwfwm5\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b5aa74e8-43d5-4cff-b520-63b6b9163130\\\",\\n  \\\"name\\\": \\\"b5aa74e8-43d5-4cff-b520-63b6b9163130\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T21:58:12.305820102Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T21:59:35.853803496Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0011866c0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260318T215943Z:3c196c67-a4c1-40eb-abc4-c5d8f5d91317\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Date\": [\n                            \"Wed, 18 Mar 2026 21:59:43 GMT\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Ms-Request-Id\": [\n                            \"e2863777-8b30-4013-8910-cf48bfc93c57\",\n                        ],\n                        \"X-Envoy-Upstream-Service-Time\": [\"11\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 694EB5D5CF094E458C2CC1B85768A697 Ref B: CHI261140510060 Ref C: 2026-03-18T21:59:43Z\",\n                        ],\n                        \"Content-Length\": [\"503\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Expires...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8a0eb43d82c81f5ac221f93e261d7d3906eccf9364890faef15f25844bdadb79","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c8702e51c9e908ed7fa0ca71653d3da76da58bbd7e3251d0f642c20febfbafb1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","failed_at":"e2e","signature_id":"9ba23246eb42a095691a46b9cbd632f747017108f5eaa0f29a11cd9f1796fc29","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0000d9d40\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-6fzwzp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ac78d6e9-5a49-4cfb-a070-1206176f3f88\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ac78d6e9-5a49-4cfb-a070-1206176f3f88\",\n      \"name\": \"ac78d6e9-5a49-4cfb-a070-1206176f3f88\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:03:11.889861811Z\",\n      \"endTime\": \"2026-03-18T22:04:50.608355115Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\\\"cluster-np-labels-taints\\\" in resourcegroup=\\\"rg-np-labels-taints-6fzwzp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ac78d6e9-5a49-4cfb-a070-1206176f3f88\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ac78d6e9-5a49-4cfb-a070-1206176f3f88\\\",\\n  \\\"name\\\": \\\"ac78d6e9-5a49-4cfb-a070-1206176f3f88\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:03:11.889861811Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:04:50.608355115Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0000d9d20\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-np-labels-taints\\\" in resourcegroup=\\\"rg-np-labels-taints-6fzwzp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ac78d6e9-5a49-4cfb-a070-1206176f3f88\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ac78d6e9-5a49-4cfb-a070-1206176f3f88\\\",\\n  \\\"name\\\": \\\"ac78d6e9-5a49-4cfb-a070-1206176f3f88\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:03:11.889861811Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:04:50.608355115Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00168adb0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Request-Id\": [\n                            \"3249ef9a-7b0f-4f55-b29c-298f2d17ac8a\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"Date\": [\n                            \"Wed, 18 Mar 2026 22:04:53 GMT\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"9\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"b917974a-e14f-4131-aa5f-6e2f9f5f3176\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"503\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=c94dda6b-80b4-4144-a102-590bf0...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0000d9d40\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-6fzwzp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ac78d6e9-5a49-4cfb-a070-1206176f3f88\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ac78d6e9-5a49-4cfb-a070-1206176f3f88\",\n      \"name\": \"ac78d6e9-5a49-4cfb-a070-1206176f3f88\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:03:11.889861811Z\",\n      \"endTime\": \"2026-03-18T22:04:50.608355115Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\\\"cluster-np-labels-taints\\\" in resourcegroup=\\\"rg-np-labels-taints-6fzwzp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ac78d6e9-5a49-4cfb-a070-1206176f3f88\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ac78d6e9-5a49-4cfb-a070-1206176f3f88\\\",\\n  \\\"name\\\": \\\"ac78d6e9-5a49-4cfb-a070-1206176f3f88\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:03:11.889861811Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:04:50.608355115Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0000d9d20\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-np-labels-taints\\\" in resourcegroup=\\\"rg-np-labels-taints-6fzwzp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ac78d6e9-5a49-4cfb-a070-1206176f3f88\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ac78d6e9-5a49-4cfb-a070-1206176f3f88\\\",\\n  \\\"name\\\": \\\"ac78d6e9-5a49-4cfb-a070-1206176f3f88\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:03:11.889861811Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:04:50.608355115Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00168adb0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Request-Id\": [\n                            \"3249ef9a-7b0f-4f55-b29c-298f2d17ac8a\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"Date\": [\n                            \"Wed, 18 Mar 2026 22:04:53 GMT\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"9\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"b917974a-e14f-4131-aa5f-6e2f9f5f3176\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"503\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=c94dda6b-80b4-4144-a102-590bf0...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8a0eb43d82c81f5ac221f93e261d7d3906eccf9364890faef15f25844bdadb79","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d14388bd0a14cfebf4af40695d0d26a1b887958b85f596f6cd57cd6e2f7b24fc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","failed_at":"e2e","signature_id":"e6aef335aed58df551668c1c4fb6e2b1c9955eae709dc370d9435a1d3a5cdffd","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0010a0360\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-xm72pt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f5cf2e2-b390-4933-a78f-5efb244ef76e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f5cf2e2-b390-4933-a78f-5efb244ef76e\",\n      \"name\": \"0f5cf2e2-b390-4933-a78f-5efb244ef76e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T21:58:05.508161468Z\",\n      \"endTime\": \"2026-03-18T21:59:35.825704307Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-xm72pt\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f5cf2e2-b390-4933-a78f-5efb244ef76e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f5cf2e2-b390-4933-a78f-5efb244ef76e\\\",\\n  \\\"name\\\": \\\"0f5cf2e2-b390-4933-a78f-5efb244ef76e\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T21:58:05.508161468Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T21:59:35.825704307Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0010a0340\u003e{\n            msg: \"failed waiting for cluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-xm72pt\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f5cf2e2-b390-4933-a78f-5efb244ef76e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f5cf2e2-b390-4933-a78f-5efb244ef76e\\\",\\n  \\\"name\\\": \\\"0f5cf2e2-b390-4933-a78f-5efb244ef76e\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T21:58:05.508161468Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T21:59:35.825704307Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000ed06c0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadaeast/9e91e3e1-c55a-41c4-9fda-c6e81031aa4e\",\n                        ],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"7892e6be-859b-443d-93ae-25f63ec15cc7\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"9\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: CBAC33FB359A45D5A9FC233D52AA531E Ref B: CHI261140506040 Ref C: 2026-03-18T21:59:37Z\",\n                        ],\n                        \"Date\": [\n                          ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0010a0360\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-xm72pt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f5cf2e2-b390-4933-a78f-5efb244ef76e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f5cf2e2-b390-4933-a78f-5efb244ef76e\",\n      \"name\": \"0f5cf2e2-b390-4933-a78f-5efb244ef76e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T21:58:05.508161468Z\",\n      \"endTime\": \"2026-03-18T21:59:35.825704307Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-xm72pt\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f5cf2e2-b390-4933-a78f-5efb244ef76e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f5cf2e2-b390-4933-a78f-5efb244ef76e\\\",\\n  \\\"name\\\": \\\"0f5cf2e2-b390-4933-a78f-5efb244ef76e\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T21:58:05.508161468Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T21:59:35.825704307Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0010a0340\u003e{\n            msg: \"failed waiting for cluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-xm72pt\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f5cf2e2-b390-4933-a78f-5efb244ef76e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f5cf2e2-b390-4933-a78f-5efb244ef76e\\\",\\n  \\\"name\\\": \\\"0f5cf2e2-b390-4933-a78f-5efb244ef76e\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T21:58:05.508161468Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T21:59:35.825704307Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000ed06c0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadaeast/9e91e3e1-c55a-41c4-9fda-c6e81031aa4e\",\n                        ],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"7892e6be-859b-443d-93ae-25f63ec15cc7\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"9\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: CBAC33FB359A45D5A9FC233D52AA531E Ref B: CHI261140506040 Ref C: 2026-03-18T21:59:37Z\",\n                        ],\n                        \"Date\": [\n                          ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8a0eb43d82c81f5ac221f93e261d7d3906eccf9364890faef15f25844bdadb79","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d8aeaaa1befa0edb96015f071234f76e04a0759909ed79b9b287420cc49919eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","failed_at":"e2e","signature_id":"237ff6b17b8ec98ad232c04a0ed44127d05d48dc83399baccc02c34cebfbfc12","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001828480\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-99xb7j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\",\n      \"name\": \"6e36926c-e6c5-49f7-a607-9f56a4bbb73b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:05:01.130113225Z\",\n      \"endTime\": \"2026-03-18T22:06:29.307118365Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Failed to verify existence of managed resource group 'patch-name-99xb7j--managed': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"failed to get quota usage data: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-99xb7j\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\\",\\n  \\\"name\\\": \\\"6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:05:01.130113225Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:06:29.307118365Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to verify existence of managed resource group 'patch-name-99xb7j--managed': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"failed to get quota usage data: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc001828460\u003e{\n            msg: \"failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-99xb7j\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\\",\\n  \\\"name\\\": \\\"6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:05:01.130113225Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:06:29.307118365Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to verify existence of managed resource group 'patch-name-99xb7j--managed': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerErro...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001828480\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-99xb7j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: MultipleErrorsOccurred\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\",\n      \"name\": \"6e36926c-e6c5-49f7-a607-9f56a4bbb73b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:05:01.130113225Z\",\n      \"endTime\": \"2026-03-18T22:06:29.307118365Z\",\n      \"error\": {\n        \"code\": \"MultipleErrorsOccurred\",\n        \"message\": \"Cluster provisioning failed due to multiple errors\",\n        \"details\": [\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"Failed to verify existence of managed resource group 'patch-name-99xb7j--managed': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"failed to get quota usage data: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          },\n          {\n            \"code\": \"InternalServerError\",\n            \"message\": \"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\n context deadline exceeded\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-99xb7j\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\\",\\n  \\\"name\\\": \\\"6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:05:01.130113225Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:06:29.307118365Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to verify existence of managed resource group 'patch-name-99xb7j--managed': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"failed to get quota usage data: ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc001828460\u003e{\n            msg: \"failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-99xb7j\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\\",\\n  \\\"name\\\": \\\"6e36926c-e6c5-49f7-a607-9f56a4bbb73b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:05:01.130113225Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:06:29.307118365Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n    \\\"message\\\": \\\"Cluster provisioning failed due to multiple errors\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"Failed to verify existence of managed resource group 'patch-name-99xb7j--managed': ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerError\\\",\\n        \\\"message\\\": \\\"ClientCertificateCredential: unable to resolve an endpoint: server response error:\\\\n context deadline exceeded\\\"\\n      },\\n      {\\n        \\\"code\\\": \\\"InternalServerErro...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ca7cc2da32e290afc8b2a3a18285c29643093d3821378dbf18bf57bc20cad012","failure_pattern":"ERROR CODE: MultipleErrorsOccurred; detail code InternalServerError; detail message ClientCertificateCredential: unable to resolve an endpoint: server response error: context deadline exceeded; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: MultipleErrorsOccurred"}},{"row_id":"d8b9ec072b63d07f8c159a017a7fa73ecdd89155809d8b2a694085fc651fbf87","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","failed_at":"e2e","signature_id":"f15a96cf1199ee286ffca5a7a3a92b706c70d32e3672375967fe435f387de27b","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0013d8460\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-lnsqh6: failed waiting for cluster=\"cluster-zstream-4-19-lnsqh6\" in resourcegroup=\"rg-zstream-upgrade-4-19-pld4fl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5ed3389f-0b4e-4794-8c26-5b3ae8de8884\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5ed3389f-0b4e-4794-8c26-5b3ae8de8884\",\n      \"name\": \"5ed3389f-0b4e-4794-8c26-5b3ae8de8884\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:09:50.25277888Z\",\n      \"endTime\": \"2026-03-18T22:11:58.43033621Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-19-lnsqh6: failed waiting for cluster=\\\"cluster-zstream-4-19-lnsqh6\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-pld4fl\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5ed3389f-0b4e-4794-8c26-5b3ae8de8884\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5ed3389f-0b4e-4794-8c26-5b3ae8de8884\\\",\\n  \\\"name\\\": \\\"5ed3389f-0b4e-4794-8c26-5b3ae8de8884\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:09:50.25277888Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:11:58.43033621Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0013d8440\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-19-lnsqh6\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-pld4fl\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5ed3389f-0b4e-4794-8c26-5b3ae8de8884\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5ed3389f-0b4e-4794-8c26-5b3ae8de8884\\\",\\n  \\\"name\\\": \\\"5ed3389f-0b4e-4794-8c26-5b3ae8de8884\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:09:50.25277888Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:11:58.43033621Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0010687e0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"748d029f-96fc-49eb-a4e6-e71ccf950083\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"X-Ms-Request-Id\": [\n                            \"54e97da7-c63e-4bd9-a811-45e592ebb83d\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 898F5B667A1D4266917476DA83548256 Ref B: CHI261140509036 Ref C: 2026-03-18T22:12:02Z\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Length\": [\"501\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadaeast/c4c65aed-8a31-40d7-afaa-2606c73c3774\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0013d8460\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-lnsqh6: failed waiting for cluster=\"cluster-zstream-4-19-lnsqh6\" in resourcegroup=\"rg-zstream-upgrade-4-19-pld4fl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5ed3389f-0b4e-4794-8c26-5b3ae8de8884\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5ed3389f-0b4e-4794-8c26-5b3ae8de8884\",\n      \"name\": \"5ed3389f-0b4e-4794-8c26-5b3ae8de8884\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:09:50.25277888Z\",\n      \"endTime\": \"2026-03-18T22:11:58.43033621Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-19-lnsqh6: failed waiting for cluster=\\\"cluster-zstream-4-19-lnsqh6\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-pld4fl\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5ed3389f-0b4e-4794-8c26-5b3ae8de8884\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5ed3389f-0b4e-4794-8c26-5b3ae8de8884\\\",\\n  \\\"name\\\": \\\"5ed3389f-0b4e-4794-8c26-5b3ae8de8884\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:09:50.25277888Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:11:58.43033621Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0013d8440\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-19-lnsqh6\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-pld4fl\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5ed3389f-0b4e-4794-8c26-5b3ae8de8884\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5ed3389f-0b4e-4794-8c26-5b3ae8de8884\\\",\\n  \\\"name\\\": \\\"5ed3389f-0b4e-4794-8c26-5b3ae8de8884\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:09:50.25277888Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:11:58.43033621Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0010687e0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"748d029f-96fc-49eb-a4e6-e71ccf950083\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"X-Ms-Request-Id\": [\n                            \"54e97da7-c63e-4bd9-a811-45e592ebb83d\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 898F5B667A1D4266917476DA83548256 Ref B: CHI261140509036 Ref C: 2026-03-18T22:12:02Z\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Length\": [\"501\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadaeast/c4c65aed-8a31-40d7-afaa-2606c73c3774\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8a0eb43d82c81f5ac221f93e261d7d3906eccf9364890faef15f25844bdadb79","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"dc2701461037065640550ff4dc27dec32513b17b4b9731b90cecf53a292a9913","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034384198847959040","occurred_at":"2026-03-18T21:39:24Z","failed_at":"e2e","signature_id":"06635d020ca291cd1c2ba88618a73ef47ef44b12a92afe822f14d0c83ccde9f6","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000962220\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-hfk97l: failed waiting for cluster=\"cluster-zstream-4-20-hfk97l\" in resourcegroup=\"rg-zstream-upgrade-4-20-6jjt78\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\",\n      \"name\": \"a535b9a7-79bf-4ae9-99b6-679319213e49\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:10:00.515143027Z\",\n      \"endTime\": \"2026-03-18T22:11:58.502121438Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-20-hfk97l: failed waiting for cluster=\\\"cluster-zstream-4-20-hfk97l\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-6jjt78\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\\\",\\n  \\\"name\\\": \\\"a535b9a7-79bf-4ae9-99b6-679319213e49\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:10:00.515143027Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:11:58.502121438Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0009621e0\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-hfk97l\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-6jjt78\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\\\",\\n  \\\"name\\\": \\\"a535b9a7-79bf-4ae9-99b6-679319213e49\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:10:00.515143027Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:11:58.502121438Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000e803c0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/a23e4d00-7c12-428e-b637-d6e87c428e63\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260318T221203Z:86a9a285-d268-4eaa-81f2-1b43b6865e4f\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"503\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"86a9a285-d268-4eaa-81f2-1b43b6865e4f\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 30110B94D9A9488595249B68A2A891EF Ref B: CHI261140505062 Ref C: 2026-03-18T22:12:03Z\",\n                        ],\n                        \"Date\": [\n                            \"Wed, 18 Mar 2026 22:12...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000962220\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-hfk97l: failed waiting for cluster=\"cluster-zstream-4-20-hfk97l\" in resourcegroup=\"rg-zstream-upgrade-4-20-6jjt78\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\",\n      \"name\": \"a535b9a7-79bf-4ae9-99b6-679319213e49\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-18T22:10:00.515143027Z\",\n      \"endTime\": \"2026-03-18T22:11:58.502121438Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-20-hfk97l: failed waiting for cluster=\\\"cluster-zstream-4-20-hfk97l\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-6jjt78\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\\\",\\n  \\\"name\\\": \\\"a535b9a7-79bf-4ae9-99b6-679319213e49\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:10:00.515143027Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:11:58.502121438Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0009621e0\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-hfk97l\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-6jjt78\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a535b9a7-79bf-4ae9-99b6-679319213e49\\\",\\n  \\\"name\\\": \\\"a535b9a7-79bf-4ae9-99b6-679319213e49\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-18T22:10:00.515143027Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-18T22:11:58.502121438Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000e803c0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/a23e4d00-7c12-428e-b637-d6e87c428e63\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260318T221203Z:86a9a285-d268-4eaa-81f2-1b43b6865e4f\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"503\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"86a9a285-d268-4eaa-81f2-1b43b6865e4f\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 30110B94D9A9488595249B68A2A891EF Ref B: CHI261140505062 Ref C: 2026-03-18T22:12:03Z\",\n                        ],\n                        \"Date\": [\n                            \"Wed, 18 Mar 2026 22:12...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8a0eb43d82c81f5ac221f93e261d7d3906eccf9364890faef15f25844bdadb79","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":22,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","8a0eb43d82c81f5ac221f93e261d7d3906eccf9364890faef15f25844bdadb79","ca156a6e6dc55851f861662371785d03e87c4fa68946334f3057fa599843ab7e","ca7cc2da32e290afc8b2a3a18285c29643093d3821378dbf18bf57bc20cad012","fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4"],"matched_occurrences":22,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034180345896636416","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-18T08:09:22Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034084482142703616","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-18T01:48:26Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034071393695436800","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-18T00:56:26Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034070234528223232","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-18T00:51:49Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034068984910843904","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-18T00:46:51Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034068629753958400","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-18T00:45:27Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2034068043331538944","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-03-18T00:43:07Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}