{"meta":{"date":"2026-03-16","timezone":"UTC","generated_at":"2026-08-30T15:06:50Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":2,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T05:06:22Z"},"failed_at":["e2e"],"failed_test_count":22,"occurrences":[{"row_id":"12f59361dd48ff29ec45852b266193c382abf6e268324f409804a12ff9eba34e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","failed_at":"e2e","signature_id":"488befe4edc4450b2b46b4d02c1793f8c3dd4e75a7c8db29f6ea6f12b2d9673b","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014454d0\u003e: \n    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5lns4q, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-5lns4q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5lns4q, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5lns4q, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-5lns4q\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5lns4q, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001459930\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5lns4q\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0014454a0\u003e{\n                msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-5lns4q\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5lns4q, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001459930\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5lns4q\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014454d0\u003e: \n    failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5lns4q, error: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-5lns4q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5lns4q, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cidr-connectivity-test, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5lns4q, error: failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-5lns4q\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5lns4q, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001459930\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5lns4q\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0014454a0\u003e{\n                msg: \"failed waiting for cluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-5lns4q\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5lns4q, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001459930\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-5lns4q\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-m559f4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-m559f4, error: context deadline exceeded"}},{"row_id":"147ee3ed5bdf442bffd4e5b1bb6a48d617b78467b79b03f687fd40ca641e6725","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","failed_at":"e2e","signature_id":"e84f3d91e58a4ddeb3430bb8931aee0419956b91f0597cc4dde0c7443f9de91f","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00101cd20\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-lpp22g, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-lpp22g\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-lpp22g, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-lpp22g, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-lpp22g\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-lpp22g, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0008a2ae0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-lpp22g\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00101ccf0\u003e{\n                msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-lpp22g\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-lpp22g, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0008a2ae0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-lpp22g\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00101cd20\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-lpp22g, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-lpp22g\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-lpp22g, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-lpp22g, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-lpp22g\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-lpp22g, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0008a2ae0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-lpp22g\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00101ccf0\u003e{\n                msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-lpp22g\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-lpp22g, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0008a2ae0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-lpp22g\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-m559f4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-m559f4, error: context deadline exceeded"}},{"row_id":"2264891e87ee7a0027b925f2d740f313c32197c0b229e5a46365147b2164e44b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","failed_at":"e2e","signature_id":"420b91d3c81d83d07724defc494257d5a7fd2f05d9cb9b50a3d99e9924d26de0","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0016292c0\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-6dmknb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d3e5222f-7626-4089-81ad-74ce5593b487\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d3e5222f-7626-4089-81ad-74ce5593b487\",\n      \"name\": \"d3e5222f-7626-4089-81ad-74ce5593b487\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-16T05:53:52.774104349Z\",\n      \"endTime\": \"2026-03-16T05:54:48.4817526Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-6dmknb\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d3e5222f-7626-4089-81ad-74ce5593b487\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d3e5222f-7626-4089-81ad-74ce5593b487\\\",\\n  \\\"name\\\": \\\"d3e5222f-7626-4089-81ad-74ce5593b487\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:53:52.774104349Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:54:48.4817526Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0016292a0\u003e{\n            msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-6dmknb\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d3e5222f-7626-4089-81ad-74ce5593b487\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d3e5222f-7626-4089-81ad-74ce5593b487\\\",\\n  \\\"name\\\": \\\"d3e5222f-7626-4089-81ad-74ce5593b487\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:53:52.774104349Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:54:48.4817526Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc001384930\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Date\": [\n                            \"Mon, 16 Mar 2026 05:54:53 GMT\",\n                        ],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/4a777810-db65-4717-9cdc-6501be453685\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260316T055453Z:b23331dc-7a87-4aac-85ec-6b1435706e75\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"501\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0016292c0\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-6dmknb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d3e5222f-7626-4089-81ad-74ce5593b487\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d3e5222f-7626-4089-81ad-74ce5593b487\",\n      \"name\": \"d3e5222f-7626-4089-81ad-74ce5593b487\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-16T05:53:52.774104349Z\",\n      \"endTime\": \"2026-03-16T05:54:48.4817526Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-6dmknb\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d3e5222f-7626-4089-81ad-74ce5593b487\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d3e5222f-7626-4089-81ad-74ce5593b487\\\",\\n  \\\"name\\\": \\\"d3e5222f-7626-4089-81ad-74ce5593b487\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:53:52.774104349Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:54:48.4817526Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0016292a0\u003e{\n            msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-6dmknb\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d3e5222f-7626-4089-81ad-74ce5593b487\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d3e5222f-7626-4089-81ad-74ce5593b487\\\",\\n  \\\"name\\\": \\\"d3e5222f-7626-4089-81ad-74ce5593b487\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:53:52.774104349Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:54:48.4817526Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc001384930\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Date\": [\n                            \"Mon, 16 Mar 2026 05:54:53 GMT\",\n                        ],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/4a777810-db65-4717-9cdc-6501be453685\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260316T055453Z:b23331dc-7a87-4aac-85ec-6b1435706e75\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"501\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8a0eb43d82c81f5ac221f93e261d7d3906eccf9364890faef15f25844bdadb79","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"28e05e67e3d3fe1c6b6751baa52d9bff8c6babeab60cfd03efa3d310aed75fc5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","failed_at":"e2e","signature_id":"3e0e267bf5b593318dc8ae60ef2869ec7a0775117cd2322f89b963a63a68c41f","test_name":"Customer should be able to create an HCP cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:96]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000695540\u003e: \n    failed to create HCP cluster basic-hcp-cluster: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-vgmwdp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8acc7e04-af91-4136-83d9-cfd4d620d5c2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8acc7e04-af91-4136-83d9-cfd4d620d5c2\",\n      \"name\": \"8acc7e04-af91-4136-83d9-cfd4d620d5c2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-16T05:58:36.120361686Z\",\n      \"endTime\": \"2026-03-16T05:59:25.865971075Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster basic-hcp-cluster: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vgmwdp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8acc7e04-af91-4136-83d9-cfd4d620d5c2\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8acc7e04-af91-4136-83d9-cfd4d620d5c2\\\",\\n  \\\"name\\\": \\\"8acc7e04-af91-4136-83d9-cfd4d620d5c2\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:58:36.120361686Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:59:25.865971075Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000695500\u003e{\n            msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vgmwdp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8acc7e04-af91-4136-83d9-cfd4d620d5c2\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8acc7e04-af91-4136-83d9-cfd4d620d5c2\\\",\\n  \\\"name\\\": \\\"8acc7e04-af91-4136-83d9-cfd4d620d5c2\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:58:36.120361686Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:59:25.865971075Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000adeed0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Request-Id\": [\n                            \"20c01661-0afe-48d2-8f91-e191ffbc139d\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260316T055926Z:6229d6a4-a060-4bcd-8adb-767c2b1c2524\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Content-Length\": [\"503\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"11\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/f93d9d56-0328-4fdd-b9c9-60f270da1dcd\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=3153...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:96]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000695540\u003e: \n    failed to create HCP cluster basic-hcp-cluster: failed waiting for cluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-vgmwdp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8acc7e04-af91-4136-83d9-cfd4d620d5c2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8acc7e04-af91-4136-83d9-cfd4d620d5c2\",\n      \"name\": \"8acc7e04-af91-4136-83d9-cfd4d620d5c2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-16T05:58:36.120361686Z\",\n      \"endTime\": \"2026-03-16T05:59:25.865971075Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster basic-hcp-cluster: failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vgmwdp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8acc7e04-af91-4136-83d9-cfd4d620d5c2\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8acc7e04-af91-4136-83d9-cfd4d620d5c2\\\",\\n  \\\"name\\\": \\\"8acc7e04-af91-4136-83d9-cfd4d620d5c2\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:58:36.120361686Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:59:25.865971075Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000695500\u003e{\n            msg: \"failed waiting for cluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-vgmwdp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8acc7e04-af91-4136-83d9-cfd4d620d5c2\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/8acc7e04-af91-4136-83d9-cfd4d620d5c2\\\",\\n  \\\"name\\\": \\\"8acc7e04-af91-4136-83d9-cfd4d620d5c2\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:58:36.120361686Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:59:25.865971075Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000adeed0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Request-Id\": [\n                            \"20c01661-0afe-48d2-8f91-e191ffbc139d\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260316T055926Z:6229d6a4-a060-4bcd-8adb-767c2b1c2524\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Content-Length\": [\"503\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"11\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/f93d9d56-0328-4fdd-b9c9-60f270da1dcd\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=3153...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8a0eb43d82c81f5ac221f93e261d7d3906eccf9364890faef15f25844bdadb79","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2cf9e2805841fa285f210581f9059ec240cd6c111ccfba0b95450e5668bc25e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","failed_at":"e2e","signature_id":"5aeff6efa147f0a4f4455f0d12d6e118c4fd736fa1d304c8919d5fa6389a770e","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00106e6f0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-jsjz55, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-jsjz55 in resource group rg-zstream-upgrade-4-21-b2bt9k, error: failed waiting for cluster=\"cluster-zstream-4-21-jsjz55\" in resourcegroup=\"rg-zstream-upgrade-4-21-b2bt9k\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-jsjz55 in resource group rg-zstream-upgrade-4-21-b2bt9k, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-21-jsjz55, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-jsjz55 in resource group rg-zstream-upgrade-4-21-b2bt9k, error: failed waiting for cluster=\\\"cluster-zstream-4-21-jsjz55\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-b2bt9k\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-jsjz55 in resource group rg-zstream-upgrade-4-21-b2bt9k, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000a91e10\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-jsjz55 in resource group rg-zstream-upgrade-4-21-b2bt9k\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00106e6c0\u003e{\n                msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-jsjz55\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-b2bt9k\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-jsjz55 in resource group rg-zstream-upgrade-4-21-b2bt9k, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000a91e10\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-jsjz55 in resource group rg-zstream-upgrade-4-21-b2bt9k\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00106e6f0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-jsjz55, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-jsjz55 in resource group rg-zstream-upgrade-4-21-b2bt9k, error: failed waiting for cluster=\"cluster-zstream-4-21-jsjz55\" in resourcegroup=\"rg-zstream-upgrade-4-21-b2bt9k\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-jsjz55 in resource group rg-zstream-upgrade-4-21-b2bt9k, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-21-jsjz55, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-jsjz55 in resource group rg-zstream-upgrade-4-21-b2bt9k, error: failed waiting for cluster=\\\"cluster-zstream-4-21-jsjz55\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-b2bt9k\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-jsjz55 in resource group rg-zstream-upgrade-4-21-b2bt9k, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000a91e10\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-jsjz55 in resource group rg-zstream-upgrade-4-21-b2bt9k\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00106e6c0\u003e{\n                msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-jsjz55\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-b2bt9k\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-jsjz55 in resource group rg-zstream-upgrade-4-21-b2bt9k, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000a91e10\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-jsjz55 in resource group rg-zstream-upgrade-4-21-b2bt9k\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-m559f4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-m559f4, error: context deadline exceeded"}},{"row_id":"420c1b69bda0b6e4c94128a6333c474d059aed56a68f9391854c37547b7ec162","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","failed_at":"e2e","signature_id":"db956ec4e40e35159a8af8607f32da52079e787390ee2e9a142f0fef2f1112c8","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0008f2c20\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-rkr4r8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n    --------------------------------------------------------------------------------\n    RESPONSE 403: 403 Forbidden\n    ERROR CODE: AuthorizationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"AuthorizationFailed\",\n        \"message\": \"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.RedHatOpenShift/hcpOpenShiftClusters/write' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128' or the scope is invalid. If access was recently granted, please refresh your credentials.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-rkr4r8\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.RedHatOpenShift/hcpOpenShiftClusters/write' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0008f2c00\u003e{\n            msg: \"failed starting cluster creation \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-rkr4r8\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.RedHatOpenShift/hcpOpenShiftClusters/write' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0014046f0\u003e{\n                ErrorCode: \"AuthorizationFailed\",\n                StatusCode: 403,\n                RawResponse: {\n                    Status: \"403 Forbidden\",\n                    StatusCode: 403,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Length\": [\"523\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"2935921b-184b-4fc2-9c09-4c26121abf36\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Date\": [\n                            \"Mon, 16 Mar 2026 05:50:31 GMT\",\n                        ],\n                        \"Content-Type\": [\n                            \"application/json; charset=utf-8\",\n                        ],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Failure-Cause\": [\"gateway\"],\n                        \"X-Ms-Request-Id\": [\n                            \"2935921b-184b-4fc2-9c09-4c26121abf36\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"EASTUS2EUAP:20260316T055032Z:2935921b-184b-4fc2-9c09-4c26121abf36\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n    ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0008f2c20\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-rkr4r8\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\n    --------------------------------------------------------------------------------\n    RESPONSE 403: 403 Forbidden\n    ERROR CODE: AuthorizationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"AuthorizationFailed\",\n        \"message\": \"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.RedHatOpenShift/hcpOpenShiftClusters/write' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128' or the scope is invalid. If access was recently granted, please refresh your credentials.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster hcp-cluster-np-128: failed starting cluster creation \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-rkr4r8\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.RedHatOpenShift/hcpOpenShiftClusters/write' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0008f2c00\u003e{\n            msg: \"failed starting cluster creation \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-rkr4r8\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.RedHatOpenShift/hcpOpenShiftClusters/write' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-rkr4r8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0014046f0\u003e{\n                ErrorCode: \"AuthorizationFailed\",\n                StatusCode: 403,\n                RawResponse: {\n                    Status: \"403 Forbidden\",\n                    StatusCode: 403,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Length\": [\"523\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"2935921b-184b-4fc2-9c09-4c26121abf36\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Date\": [\n                            \"Mon, 16 Mar 2026 05:50:31 GMT\",\n                        ],\n                        \"Content-Type\": [\n                            \"application/json; charset=utf-8\",\n                        ],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Failure-Cause\": [\"gateway\"],\n                        \"X-Ms-Request-Id\": [\n                            \"2935921b-184b-4fc2-9c09-4c26121abf36\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"EASTUS2EUAP:20260316T055032Z:2935921b-184b-4fc2-9c09-4c26121abf36\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n    ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"eff4ac5ab820367904c1fb3e78845caba542fd4825dfa1c42a1fdcf6260aa246","failure_pattern":"ERROR CODE: AuthorizationFailed; detail message The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '\u003cuuid\u003e' does not have authorization to perform action 'Microsoft.RedHatOpenShift/hcpOpenShiftClusters/\u003cresource\u003e' over scope '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128' or the scope is invalid. If access was recently granted, please refresh your credentials.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: AuthorizationFailed"}},{"row_id":"43c885e166af1ea11c48075a44bcd1f864a75a4482d899df3269089290e8ad20","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","failed_at":"e2e","signature_id":"1eb3805a5fe71ec50edd4d69c587ec8350ab24f91e053ebfd61eded6cdf1d4ad","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ddc570\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-fsqvnh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-fsqvnh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0013818e0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000ddc540\u003e{\n                msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-fsqvnh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0013818e0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ddc570\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-fsqvnh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-fsqvnh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0013818e0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000ddc540\u003e{\n                msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-fsqvnh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0013818e0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-fsqvnh\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-m559f4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-m559f4, error: context deadline exceeded"}},{"row_id":"517be7456fc788b111131ca0e9a90ff07209834cdcbb287e99bcd3af53c6f9b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","failed_at":"e2e","signature_id":"a3917a20697f535d483e2f50596dc25ed08eb3cecf016c87a084b621b064671c","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e27920\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-m559f4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-m559f4, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-m559f4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-m559f4, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000741990\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-m559f4\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e27920\u003e: \n    failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-m559f4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-m559f4, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-m559f4\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-m559f4, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000741990\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-m559f4\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-m559f4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-m559f4, error: context deadline exceeded"}},{"row_id":"6a8aed4490272b77e17a010b168f0b32c9dcc611d4d1fe761809de9ca366b1ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","failed_at":"e2e","signature_id":"0c2eda5743cefd6dcfe9ecf740d39893dbf60d284ed5aac300adeb234f564f8f","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00095c4c0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-psrj8k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b4b3429d-a139-425f-b110-f68346a4fe8b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b4b3429d-a139-425f-b110-f68346a4fe8b\",\n      \"name\": \"b4b3429d-a139-425f-b110-f68346a4fe8b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-16T05:54:04.320816036Z\",\n      \"endTime\": \"2026-03-16T05:54:48.517209203Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-psrj8k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b4b3429d-a139-425f-b110-f68346a4fe8b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b4b3429d-a139-425f-b110-f68346a4fe8b\\\",\\n  \\\"name\\\": \\\"b4b3429d-a139-425f-b110-f68346a4fe8b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:54:04.320816036Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:54:48.517209203Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00095c4a0\u003e{\n            msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-psrj8k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b4b3429d-a139-425f-b110-f68346a4fe8b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b4b3429d-a139-425f-b110-f68346a4fe8b\\\",\\n  \\\"name\\\": \\\"b4b3429d-a139-425f-b110-f68346a4fe8b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:54:04.320816036Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:54:48.517209203Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000974690\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Length\": [\"503\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Request-Id\": [\n                            \"39e1d21e-9cf9-413f-8981-bd67e2dcc0da\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"6c4b46e2-9a40-4232-9e7e-8cba90b18fb2\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/57085293-a6f4-4a48-a611-c5bc14532134\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"m...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00095c4c0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-psrj8k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b4b3429d-a139-425f-b110-f68346a4fe8b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b4b3429d-a139-425f-b110-f68346a4fe8b\",\n      \"name\": \"b4b3429d-a139-425f-b110-f68346a4fe8b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-16T05:54:04.320816036Z\",\n      \"endTime\": \"2026-03-16T05:54:48.517209203Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-psrj8k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b4b3429d-a139-425f-b110-f68346a4fe8b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b4b3429d-a139-425f-b110-f68346a4fe8b\\\",\\n  \\\"name\\\": \\\"b4b3429d-a139-425f-b110-f68346a4fe8b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:54:04.320816036Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:54:48.517209203Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00095c4a0\u003e{\n            msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-psrj8k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b4b3429d-a139-425f-b110-f68346a4fe8b\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b4b3429d-a139-425f-b110-f68346a4fe8b\\\",\\n  \\\"name\\\": \\\"b4b3429d-a139-425f-b110-f68346a4fe8b\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:54:04.320816036Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:54:48.517209203Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000974690\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Content-Length\": [\"503\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Request-Id\": [\n                            \"39e1d21e-9cf9-413f-8981-bd67e2dcc0da\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"6c4b46e2-9a40-4232-9e7e-8cba90b18fb2\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/57085293-a6f4-4a48-a611-c5bc14532134\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"m...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8a0eb43d82c81f5ac221f93e261d7d3906eccf9364890faef15f25844bdadb79","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8121dfa8764227731e9f172c444d7769087f9e58c62d7bd70b1436109c97e17f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","failed_at":"e2e","signature_id":"6970bcc652bfcfa5b97b4fc669aea33f82002482e6faf25e9f132a1466275393","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001220440\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-slfr7d: failed waiting for cluster=\"cluster-zstream-4-19-slfr7d\" in resourcegroup=\"rg-zstream-upgrade-4-19-wkgsx5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bbd60cd6-e44d-4a7e-a230-ef5e2ef66226\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bbd60cd6-e44d-4a7e-a230-ef5e2ef66226\",\n      \"name\": \"bbd60cd6-e44d-4a7e-a230-ef5e2ef66226\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-16T06:04:10.39702426Z\",\n      \"endTime\": \"2026-03-16T06:05:14.283265287Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-19-slfr7d: failed waiting for cluster=\\\"cluster-zstream-4-19-slfr7d\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-wkgsx5\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bbd60cd6-e44d-4a7e-a230-ef5e2ef66226\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bbd60cd6-e44d-4a7e-a230-ef5e2ef66226\\\",\\n  \\\"name\\\": \\\"bbd60cd6-e44d-4a7e-a230-ef5e2ef66226\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T06:04:10.39702426Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T06:05:14.283265287Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc001220420\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-19-slfr7d\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-wkgsx5\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bbd60cd6-e44d-4a7e-a230-ef5e2ef66226\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bbd60cd6-e44d-4a7e-a230-ef5e2ef66226\\\",\\n  \\\"name\\\": \\\"bbd60cd6-e44d-4a7e-a230-ef5e2ef66226\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T06:04:10.39702426Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T06:05:14.283265287Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc001388720\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Length\": [\"502\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Request-Id\": [\n                            \"3df2300f-9029-4412-ba6b-ef329877e1a7\",\n                        ],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/2b9b79d1-95be-4d60-9c5c-30c7bd925b0e\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"8\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"6a3bc74e-3765-48a4-a8c2-26ff92d85256\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                        ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001220440\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-slfr7d: failed waiting for cluster=\"cluster-zstream-4-19-slfr7d\" in resourcegroup=\"rg-zstream-upgrade-4-19-wkgsx5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bbd60cd6-e44d-4a7e-a230-ef5e2ef66226\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bbd60cd6-e44d-4a7e-a230-ef5e2ef66226\",\n      \"name\": \"bbd60cd6-e44d-4a7e-a230-ef5e2ef66226\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-16T06:04:10.39702426Z\",\n      \"endTime\": \"2026-03-16T06:05:14.283265287Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-19-slfr7d: failed waiting for cluster=\\\"cluster-zstream-4-19-slfr7d\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-wkgsx5\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bbd60cd6-e44d-4a7e-a230-ef5e2ef66226\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bbd60cd6-e44d-4a7e-a230-ef5e2ef66226\\\",\\n  \\\"name\\\": \\\"bbd60cd6-e44d-4a7e-a230-ef5e2ef66226\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T06:04:10.39702426Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T06:05:14.283265287Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc001220420\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-19-slfr7d\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-wkgsx5\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bbd60cd6-e44d-4a7e-a230-ef5e2ef66226\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/bbd60cd6-e44d-4a7e-a230-ef5e2ef66226\\\",\\n  \\\"name\\\": \\\"bbd60cd6-e44d-4a7e-a230-ef5e2ef66226\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T06:04:10.39702426Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T06:05:14.283265287Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc001388720\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Length\": [\"502\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Request-Id\": [\n                            \"3df2300f-9029-4412-ba6b-ef329877e1a7\",\n                        ],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/2b9b79d1-95be-4d60-9c5c-30c7bd925b0e\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"8\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"6a3bc74e-3765-48a4-a8c2-26ff92d85256\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                        ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8a0eb43d82c81f5ac221f93e261d7d3906eccf9364890faef15f25844bdadb79","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"85c3067213b170caf3f278c114c1dd6e98e4a90be7cd441db42d7cf45efc5129","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","failed_at":"e2e","signature_id":"f84db3c137cad46a28c3e54119e47a5ebf7698ff6bc2244e20e068bd0436bdd4","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a4cea0\u003e: \n    failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-b5b8bw, error: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-b5b8bw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-b5b8bw, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-b5b8bw, error: failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-b5b8bw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-b5b8bw, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0015a98c0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-b5b8bw\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000a4cae0\u003e{\n                msg: \"failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-b5b8bw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-b5b8bw, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0015a98c0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-b5b8bw\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a4cea0\u003e: \n    failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-b5b8bw, error: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-b5b8bw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-b5b8bw, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-b5b8bw, error: failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-b5b8bw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-b5b8bw, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0015a98c0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-b5b8bw\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000a4cae0\u003e{\n                msg: \"failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-b5b8bw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-b5b8bw, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0015a98c0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-b5b8bw\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-m559f4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-m559f4, error: context deadline exceeded"}},{"row_id":"87ef33b0f73e6ea10292c7a6b19fc5ecad7951548d86b00ecad9372336210cdf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","failed_at":"e2e","signature_id":"994949bc78b1800725718804f81191e64a987b1448eba49bcdbdb8afecca0b76","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0007a0f20\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-j5p24k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2d2e9797-4414-473e-b4af-5fa3295c39cc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2d2e9797-4414-473e-b4af-5fa3295c39cc\",\n      \"name\": \"2d2e9797-4414-473e-b4af-5fa3295c39cc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-16T05:58:43.117719355Z\",\n      \"endTime\": \"2026-03-16T05:59:25.785868644Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-j5p24k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2d2e9797-4414-473e-b4af-5fa3295c39cc\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2d2e9797-4414-473e-b4af-5fa3295c39cc\\\",\\n  \\\"name\\\": \\\"2d2e9797-4414-473e-b4af-5fa3295c39cc\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:58:43.117719355Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:59:25.785868644Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0007a0ee0\u003e{\n            msg: \"failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-j5p24k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2d2e9797-4414-473e-b4af-5fa3295c39cc\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2d2e9797-4414-473e-b4af-5fa3295c39cc\\\",\\n  \\\"name\\\": \\\"2d2e9797-4414-473e-b4af-5fa3295c39cc\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:58:43.117719355Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:59:25.785868644Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000ea4b10\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Request-Id\": [\n                            \"f54f88d5-3cde-4dbc-af86-37ec91b2f129\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"b6dae1aa-ff73-4fd1-815f-9bb691862adb\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260316T055934Z:b6dae1aa-ff73-4fd1-815f-9bb691862adb\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/9a86779f-f19f-4090-82c7-e45be2465d60\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 2F3009F4FC7845318C8D158FD658CC85 Ref B: CHI2611...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:89]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0007a0f20\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-j5p24k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2d2e9797-4414-473e-b4af-5fa3295c39cc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2d2e9797-4414-473e-b4af-5fa3295c39cc\",\n      \"name\": \"2d2e9797-4414-473e-b4af-5fa3295c39cc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-16T05:58:43.117719355Z\",\n      \"endTime\": \"2026-03-16T05:59:25.785868644Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-j5p24k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2d2e9797-4414-473e-b4af-5fa3295c39cc\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2d2e9797-4414-473e-b4af-5fa3295c39cc\\\",\\n  \\\"name\\\": \\\"2d2e9797-4414-473e-b4af-5fa3295c39cc\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:58:43.117719355Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:59:25.785868644Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0007a0ee0\u003e{\n            msg: \"failed waiting for cluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-j5p24k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2d2e9797-4414-473e-b4af-5fa3295c39cc\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/2d2e9797-4414-473e-b4af-5fa3295c39cc\\\",\\n  \\\"name\\\": \\\"2d2e9797-4414-473e-b4af-5fa3295c39cc\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:58:43.117719355Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:59:25.785868644Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000ea4b10\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Request-Id\": [\n                            \"f54f88d5-3cde-4dbc-af86-37ec91b2f129\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"b6dae1aa-ff73-4fd1-815f-9bb691862adb\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260316T055934Z:b6dae1aa-ff73-4fd1-815f-9bb691862adb\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/9a86779f-f19f-4090-82c7-e45be2465d60\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 2F3009F4FC7845318C8D158FD658CC85 Ref B: CHI2611...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8a0eb43d82c81f5ac221f93e261d7d3906eccf9364890faef15f25844bdadb79","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"89084f84456fcb704aecb1928970c39403bd9d2ba452560d92a3912c2d10041e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","failed_at":"e2e","signature_id":"d96df068ad434e9a0e2966e4bb5122791524d6797a32adea9a72c57456b99c47","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00103ca80\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-6m66gz, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-6m66gz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-6m66gz, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-6m66gz, error: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-6m66gz\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-6m66gz, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0008dddb0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-6m66gz\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00103ca50\u003e{\n                msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-6m66gz\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-6m66gz, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0008dddb0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-6m66gz\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00103ca80\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-6m66gz, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-6m66gz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-6m66gz, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-6m66gz, error: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-6m66gz\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-6m66gz, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0008dddb0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-6m66gz\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00103ca50\u003e{\n                msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-6m66gz\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-6m66gz, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0008dddb0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-6m66gz\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-m559f4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-m559f4, error: context deadline exceeded"}},{"row_id":"8940de19abc0bd567573ffdc111993d6fa54024ef903c2740e78875ae4437148","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","failed_at":"e2e","signature_id":"f5fb0b3b4bff165498e2d9ddacd8e71b63b31ca295b435d5432bfc3cf1966be7","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:81]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001504fc0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-basic-hcp-cluster-7hz8zk\" in resourcegroup=\"customer-rg-clsh4k\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609\n    --------------------------------------------------------------------------------\n    RESPONSE 403: 403 Forbidden\n    ERROR CODE: AuthorizationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"AuthorizationFailed\",\n        \"message\": \"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Resources/deployments/operationStatuses/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609' or the scope is invalid. If access was recently granted, please refresh your credentials.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-basic-hcp-cluster-7hz8zk\\\" in resourcegroup=\\\"customer-rg-clsh4k\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Resources/deployments/operationStatuses/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc001504fa0\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-basic-hcp-cluster-7hz8zk\\\" in resourcegroup=\\\"customer-rg-clsh4k\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Resources/deployments/operationStatuses/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0014bb1a0\u003e{\n                ErrorCode: \"AuthorizationFailed\",\n                StatusCode: 403,\n                RawResponse: {\n                    Status: \"403 Forbidden\",\n                    StatusCode: 403,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Failure-Cause\": [\"gateway\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"06490acc-bbf6-4428-8144-e61db004ace3\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"EASTUS2EUAP:20260316T060848Z:06490acc-bbf6-4428-8144-e61db004ace3\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"569\"],\n                        \"Content-Type\": [\n                            \"applic...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:81]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001504fc0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-basic-hcp-cluster-7hz8zk\" in resourcegroup=\"customer-rg-clsh4k\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609\n    --------------------------------------------------------------------------------\n    RESPONSE 403: 403 Forbidden\n    ERROR CODE: AuthorizationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"AuthorizationFailed\",\n        \"message\": \"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Resources/deployments/operationStatuses/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609' or the scope is invalid. If access was recently granted, please refresh your credentials.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-basic-hcp-cluster-7hz8zk\\\" in resourcegroup=\\\"customer-rg-clsh4k\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Resources/deployments/operationStatuses/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc001504fa0\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-basic-hcp-cluster-7hz8zk\\\" in resourcegroup=\\\"customer-rg-clsh4k\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Resources/deployments/operationStatuses/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/customer-rg-clsh4k/providers/Microsoft.Resources/deployments/customer-infra-basic-hcp-cluster-7hz8zk/operationStatuses/08584279655981786609' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0014bb1a0\u003e{\n                ErrorCode: \"AuthorizationFailed\",\n                StatusCode: 403,\n                RawResponse: {\n                    Status: \"403 Forbidden\",\n                    StatusCode: 403,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Failure-Cause\": [\"gateway\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"06490acc-bbf6-4428-8144-e61db004ace3\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"EASTUS2EUAP:20260316T060848Z:06490acc-bbf6-4428-8144-e61db004ace3\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"569\"],\n                        \"Content-Type\": [\n                            \"applic...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"893592381c30302c0ad58ee9d8f6dc46d73953b616076e02ee3cc8d0f8a1abef","failure_pattern":"ERROR CODE: AuthorizationFailed","search_query":"ERROR CODE: AuthorizationFailed"}},{"row_id":"94cec306fce76b02cc8220240ef19081f6035a19b5548c00ed71802e84c706fc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","failed_at":"e2e","signature_id":"aa224806ca603d56173bc8c4551b5efd8e98c6c2cdc24d4ee7cb97d7e3e4866e","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:74]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00093d060\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-np-autoscale-cluster-5pp8qh\" in resourcegroup=\"np-autoscaling-l7brcq\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Resources/deployments/customer-infra-np-autoscale-cluster-5pp8qh/operationStatuses/08584279658982549597\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Forbidden\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\r\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Network/virtualNetworks/customer-vnet' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-np-autoscale-cluster-5pp8qh\\\" in resourcegroup=\\\"np-autoscaling-l7brcq\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Resources/deployments/customer-infra-np-autoscale-cluster-5pp8qh/operationStatuses/08584279658982549597\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Network/virtualNetworks/customer-vnet' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00093d020\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-np-autoscale-cluster-5pp8qh\\\" in resourcegroup=\\\"np-autoscaling-l7brcq\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Resources/deployments/customer-infra-np-autoscale-cluster-5pp8qh/operationStatuses/08584279658982549597\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Network/virtualNetworks/customer-vnet' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0008efc50\u003e{\n                ErrorCode: \"DeploymentFailed\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: ED37EB9E1FA645FA954684A176851865 Ref B: CH1AA2020620045 Ref C: 2026-03-16T06:05:39Z\",\n                        ],\n                        \"Date\": [\n        ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:74]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00093d060\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-np-autoscale-cluster-5pp8qh\" in resourcegroup=\"np-autoscaling-l7brcq\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Resources/deployments/customer-infra-np-autoscale-cluster-5pp8qh/operationStatuses/08584279658982549597\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Forbidden\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\r\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Network/virtualNetworks/customer-vnet' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-np-autoscale-cluster-5pp8qh\\\" in resourcegroup=\\\"np-autoscaling-l7brcq\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Resources/deployments/customer-infra-np-autoscale-cluster-5pp8qh/operationStatuses/08584279658982549597\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Network/virtualNetworks/customer-vnet' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00093d020\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-np-autoscale-cluster-5pp8qh\\\" in resourcegroup=\\\"np-autoscaling-l7brcq\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Resources/deployments/customer-infra-np-autoscale-cluster-5pp8qh/operationStatuses/08584279658982549597\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/np-autoscaling-l7brcq/providers/Microsoft.Network/virtualNetworks/customer-vnet' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0008efc50\u003e{\n                ErrorCode: \"DeploymentFailed\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: ED37EB9E1FA645FA954684A176851865 Ref B: CH1AA2020620045 Ref C: 2026-03-16T06:05:39Z\",\n                        ],\n                        \"Date\": [\n        ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1ab4a070713c2799384044a1c8b07d256e44b43cd64be86c9bc5395d9df5c28e","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AuthorizationFailed; detail message The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '\u003cuuid\u003e' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/\u003cresource\u003e' over scope '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/virtualNetworks/customer-vnet' or the scope is invalid. If access was recently granted, please refresh your credentials.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"9701149ac59ccbd0fec17be013905a32523c4c1cc10d3360814031f214951c01","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","failed_at":"e2e","signature_id":"3b352a408296d3a0251d7a6d14862960baca6761c1863c44d92551e765866899","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:83]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00144c090\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5, error: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-l82rz5\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5, error: failed waiting for cluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-l82rz5\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000aed740\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00144c060\u003e{\n                msg: \"failed waiting for cluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-l82rz5\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000aed740\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:83]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00144c090\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5, error: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-l82rz5\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5, error: failed waiting for cluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-l82rz5\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000aed740\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00144c060\u003e{\n                msg: \"failed waiting for cluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-l82rz5\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000aed740\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-l82rz5\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-m559f4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-m559f4, error: context deadline exceeded"}},{"row_id":"9854bd11e978f105c76a83fa0c1105c6c4527b363bdedc36188e0a5869ae186f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","failed_at":"e2e","signature_id":"0e70a576c756dcd421b0a13e1846e4a5a70d8e4db210c128bf73cd9a2927613f","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00088aac0\u003e: \n    failed to create HCP cluster ea-listnpt7gm: failed waiting for cluster=\"ea-listnpt7gm\" in resourcegroup=\"ea-list-zq4lft\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/09a3be35-a51e-474a-8388-36b58cf61508\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/09a3be35-a51e-474a-8388-36b58cf61508\",\n      \"name\": \"09a3be35-a51e-474a-8388-36b58cf61508\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-16T06:00:58.543305109Z\",\n      \"endTime\": \"2026-03-16T06:01:21.462955449Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-listnpt7gm: failed waiting for cluster=\\\"ea-listnpt7gm\\\" in resourcegroup=\\\"ea-list-zq4lft\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/09a3be35-a51e-474a-8388-36b58cf61508\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/09a3be35-a51e-474a-8388-36b58cf61508\\\",\\n  \\\"name\\\": \\\"09a3be35-a51e-474a-8388-36b58cf61508\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T06:00:58.543305109Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T06:01:21.462955449Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00088aaa0\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-listnpt7gm\\\" in resourcegroup=\\\"ea-list-zq4lft\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/09a3be35-a51e-474a-8388-36b58cf61508\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/09a3be35-a51e-474a-8388-36b58cf61508\\\",\\n  \\\"name\\\": \\\"09a3be35-a51e-474a-8388-36b58cf61508\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T06:00:58.543305109Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T06:01:21.462955449Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00077bbc0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/381463b8-de5e-41cf-a1d6-b149c681d8c1\",\n                        ],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"a99d0b49-b6e9-413e-830a-2dcdf4b4a62d\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260316T060129Z:a99d0b49-b6e9-413e-830a-2dcdf4b4a62d\",\n                        ],\n                        \"X-Msedge-Ref\": [\n  ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00088aac0\u003e: \n    failed to create HCP cluster ea-listnpt7gm: failed waiting for cluster=\"ea-listnpt7gm\" in resourcegroup=\"ea-list-zq4lft\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/09a3be35-a51e-474a-8388-36b58cf61508\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/09a3be35-a51e-474a-8388-36b58cf61508\",\n      \"name\": \"09a3be35-a51e-474a-8388-36b58cf61508\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-16T06:00:58.543305109Z\",\n      \"endTime\": \"2026-03-16T06:01:21.462955449Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster ea-listnpt7gm: failed waiting for cluster=\\\"ea-listnpt7gm\\\" in resourcegroup=\\\"ea-list-zq4lft\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/09a3be35-a51e-474a-8388-36b58cf61508\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/09a3be35-a51e-474a-8388-36b58cf61508\\\",\\n  \\\"name\\\": \\\"09a3be35-a51e-474a-8388-36b58cf61508\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T06:00:58.543305109Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T06:01:21.462955449Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00088aaa0\u003e{\n            msg: \"failed waiting for cluster=\\\"ea-listnpt7gm\\\" in resourcegroup=\\\"ea-list-zq4lft\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/09a3be35-a51e-474a-8388-36b58cf61508\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/09a3be35-a51e-474a-8388-36b58cf61508\\\",\\n  \\\"name\\\": \\\"09a3be35-a51e-474a-8388-36b58cf61508\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T06:00:58.543305109Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T06:01:21.462955449Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00077bbc0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/381463b8-de5e-41cf-a1d6-b149c681d8c1\",\n                        ],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"a99d0b49-b6e9-413e-830a-2dcdf4b4a62d\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260316T060129Z:a99d0b49-b6e9-413e-830a-2dcdf4b4a62d\",\n                        ],\n                        \"X-Msedge-Ref\": [\n  ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8a0eb43d82c81f5ac221f93e261d7d3906eccf9364890faef15f25844bdadb79","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b6f466954e4680bc984e92e78766fe55bcda116423c53145ede5bcdded2ccc95","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","failed_at":"e2e","signature_id":"27453cdf92c9666ede34c49454a77bb28518b6f6a37d519d92e216ae52bf28bc","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:96]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000760480\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-basic-cluster-wk5cgh\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-dvvbjq\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Resources/deployments/customer-infra-basic-cluster-wk5cgh/operationStatuses/08584279668585153729\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Forbidden\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\r\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-basic-cluster-wk5cgh\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-dvvbjq\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Resources/deployments/customer-infra-basic-cluster-wk5cgh/operationStatuses/08584279668585153729\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000760460\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-basic-cluster-wk5cgh\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-dvvbjq\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Resources/deployments/customer-infra-basic-cluster-wk5cgh/operationStatuses/08584279668585153729\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0012bd110\u003e{\n                ErrorCode: \"DeploymentFailed\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Length\": [\"839\"],\n                 ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:96]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000760480\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-basic-cluster-wk5cgh\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-dvvbjq\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Resources/deployments/customer-infra-basic-cluster-wk5cgh/operationStatuses/08584279668585153729\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Forbidden\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\r\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-basic-cluster-wk5cgh\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-dvvbjq\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Resources/deployments/customer-infra-basic-cluster-wk5cgh/operationStatuses/08584279668585153729\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000760460\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-basic-cluster-wk5cgh\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-dvvbjq\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Resources/deployments/customer-infra-basic-cluster-wk5cgh/operationStatuses/08584279668585153729\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-dvvbjq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0012bd110\u003e{\n                ErrorCode: \"DeploymentFailed\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Length\": [\"839\"],\n                 ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6461ea09e6946cd19f6c887ff25cabbd85071eb5cb02d829439f92ba37e09b36","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AuthorizationFailed; detail message The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '\u003cuuid\u003e' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/\u003cresource\u003e' over scope '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1' or the scope is invalid. If access was recently granted, please refresh your credentials.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"bd90099df0cb6b6ebecfac7a16131166bc9c9ab60d408d6c47c151f4db92154b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","failed_at":"e2e","signature_id":"b8285d1fe5e0e06b0118077c74385230269c41bde5a189b9513acacc0d1a338e","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes"}},{"row_id":"e176db3f798ddc09685306b3cc30a38d6d4ff0845fbb64d7583d5f36a7345313","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","failed_at":"e2e","signature_id":"f4dcf9151c86716fa85f7d2354f36e247bbe355f6542caad32346c5877430eb7","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00122f260\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-jm9qgg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6998cd22-db46-45f1-aa33-43755a778f3f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6998cd22-db46-45f1-aa33-43755a778f3f\",\n      \"name\": \"6998cd22-db46-45f1-aa33-43755a778f3f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-16T05:57:41.628783027Z\",\n      \"endTime\": \"2026-03-16T05:58:24.131373366Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-jm9qgg\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6998cd22-db46-45f1-aa33-43755a778f3f\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6998cd22-db46-45f1-aa33-43755a778f3f\\\",\\n  \\\"name\\\": \\\"6998cd22-db46-45f1-aa33-43755a778f3f\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:57:41.628783027Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:58:24.131373366Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00122f240\u003e{\n            msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-jm9qgg\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6998cd22-db46-45f1-aa33-43755a778f3f\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6998cd22-db46-45f1-aa33-43755a778f3f\\\",\\n  \\\"name\\\": \\\"6998cd22-db46-45f1-aa33-43755a778f3f\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:57:41.628783027Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:58:24.131373366Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000915ad0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"8e59cc34-80cf-4dd7-a7a4-e8bba1de4a86\",\n                        ],\n                        \"Date\": [\n                            \"Mon, 16 Mar 2026 05:58:32 GMT\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/8c536465-a5dc-4067-a393-690861875e85\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"503\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260316T055832Z:8e59cc34-80cf-4dd7-a7a4-e8bba...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00122f260\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-jm9qgg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6998cd22-db46-45f1-aa33-43755a778f3f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6998cd22-db46-45f1-aa33-43755a778f3f\",\n      \"name\": \"6998cd22-db46-45f1-aa33-43755a778f3f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-03-16T05:57:41.628783027Z\",\n      \"endTime\": \"2026-03-16T05:58:24.131373366Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-jm9qgg\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6998cd22-db46-45f1-aa33-43755a778f3f\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6998cd22-db46-45f1-aa33-43755a778f3f\\\",\\n  \\\"name\\\": \\\"6998cd22-db46-45f1-aa33-43755a778f3f\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:57:41.628783027Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:58:24.131373366Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00122f240\u003e{\n            msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-jm9qgg\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6998cd22-db46-45f1-aa33-43755a778f3f\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6998cd22-db46-45f1-aa33-43755a778f3f\\\",\\n  \\\"name\\\": \\\"6998cd22-db46-45f1-aa33-43755a778f3f\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-03-16T05:57:41.628783027Z\\\",\\n  \\\"endTime\\\": \\\"2026-03-16T05:58:24.131373366Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000915ad0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"8e59cc34-80cf-4dd7-a7a4-e8bba1de4a86\",\n                        ],\n                        \"Date\": [\n                            \"Mon, 16 Mar 2026 05:58:32 GMT\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/8c536465-a5dc-4067-a393-690861875e85\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"503\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260316T055832Z:8e59cc34-80cf-4dd7-a7a4-e8bba...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8a0eb43d82c81f5ac221f93e261d7d3906eccf9364890faef15f25844bdadb79","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f99bc858773093b10271a50aed0f5cf712b90e282098df20741727bc6020b6df","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","failed_at":"e2e","signature_id":"08e328dc1dfd1fb161e5ee5fbeb36a333526cb48a4a6251392be2ebade86d839","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00155d560\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-vrd9tg, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr, error: failed waiting for cluster=\"cluster-zstream-4-20-vrd9tg\" in resourcegroup=\"rg-zstream-upgrade-4-20-vppdwr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-20-vrd9tg, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr, error: failed waiting for cluster=\\\"cluster-zstream-4-20-vrd9tg\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-vppdwr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000a12410\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00155d530\u003e{\n                msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-vrd9tg\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-vppdwr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000a12410\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:95]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00155d560\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-vrd9tg, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr, error: failed waiting for cluster=\"cluster-zstream-4-20-vrd9tg\" in resourcegroup=\"rg-zstream-upgrade-4-20-vppdwr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-20-vrd9tg, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr, error: failed waiting for cluster=\\\"cluster-zstream-4-20-vrd9tg\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-vppdwr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000a12410\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00155d530\u003e{\n                msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-vrd9tg\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-vppdwr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000a12410\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-vrd9tg in resource group rg-zstream-upgrade-4-20-vppdwr\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-m559f4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-m559f4, error: context deadline exceeded"}},{"row_id":"fee048333bf1e1afc549cdb3423aeb633aa0a424233e1618df821e8141fcd598","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033409517772869632","occurred_at":"2026-03-16T05:06:22Z","failed_at":"e2e","signature_id":"a10960aad13ed9377a917637cf1712dc7c65ca2dd34b3f121b55ef5f9935f1e0","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0004993c0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-ea-cluster-lqkgx2\" in resourcegroup=\"external-auth-cluster-rhfph2\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Resources/deployments/customer-infra-ea-cluster-lqkgx2/operationStatuses/08584279665118667979\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Forbidden\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\r\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-ea-cluster-lqkgx2\\\" in resourcegroup=\\\"external-auth-cluster-rhfph2\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Resources/deployments/customer-infra-ea-cluster-lqkgx2/operationStatuses/08584279665118667979\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000499380\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-ea-cluster-lqkgx2\\\" in resourcegroup=\\\"external-auth-cluster-rhfph2\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Resources/deployments/customer-infra-ea-cluster-lqkgx2/operationStatuses/08584279665118667979\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000baa6f0\u003e{\n                ErrorCode: \"DeploymentFailed\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n     ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0004993c0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-ea-cluster-lqkgx2\" in resourcegroup=\"external-auth-cluster-rhfph2\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Resources/deployments/customer-infra-ea-cluster-lqkgx2/operationStatuses/08584279665118667979\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Forbidden\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"AuthorizationFailed\\\",\\r\\n    \\\"message\\\": \\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create customer-infra: failed waiting for deployment \\\"customer-infra-ea-cluster-lqkgx2\\\" in resourcegroup=\\\"external-auth-cluster-rhfph2\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Resources/deployments/customer-infra-ea-cluster-lqkgx2/operationStatuses/08584279665118667979\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000499380\u003e{\n            msg: \"failed waiting for deployment \\\"customer-infra-ea-cluster-lqkgx2\\\" in resourcegroup=\\\"external-auth-cluster-rhfph2\\\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Resources/deployments/customer-infra-ea-cluster-lqkgx2/operationStatuses/08584279665118667979\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Forbidden\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '0e96dbb7-17a2-4393-b2d3-d84524e39c32' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/external-auth-cluster-rhfph2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc000baa6f0\u003e{\n                ErrorCode: \"DeploymentFailed\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n     ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3efdb49df868badec71fe7b3172881c5f208bc6b8873fb2baedf7ca51b8593a4","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AuthorizationFailed; detail message The client 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' with object id '\u003cuuid\u003e' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/\u003cresource\u003e' over scope '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":22,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","1ab4a070713c2799384044a1c8b07d256e44b43cd64be86c9bc5395d9df5c28e","1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","3efdb49df868badec71fe7b3172881c5f208bc6b8873fb2baedf7ca51b8593a4","6461ea09e6946cd19f6c887ff25cabbd85071eb5cb02d829439f92ba37e09b36","893592381c30302c0ad58ee9d8f6dc46d73953b616076e02ee3cc8d0f8a1abef","8a0eb43d82c81f5ac221f93e261d7d3906eccf9364890faef15f25844bdadb79","eff4ac5ab820367904c1fb3e78845caba542fd4825dfa1c42a1fdcf6260aa246"],"matched_occurrences":22,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033362659054718976","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-03-16T02:00:10Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"eb7311c51d9912703a7fca7842b3f05b5d32d14276102238ab7f090c38084cbf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2033362659054718976","occurred_at":"2026-03-16T02:00:10Z","failed_at":"e2e","signature_id":"69b6d65c2a309f09c806b140114cf9bcb0384b2a18f675177e552539baa0bdd1","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:60]: Timed out after 600.007s.\nNFD operator should be installed successfully\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001039e40\u003e: \n    subscription openshift-nfd/nfd not ready, state=\"\" (OLM still processing, no errors detected)\n    {\n        s: \"subscription openshift-nfd/nfd not ready, state=\\\"\\\" (OLM still processing, no errors detected)\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:60]: Timed out after 600.007s.\nNFD operator should be installed successfully\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001039e40\u003e: \n    subscription openshift-nfd/nfd not ready, state=\"\" (OLM still processing, no errors detected)\n    {\n        s: \"subscription openshift-nfd/nfd not ready, state=\\\"\\\" (OLM still processing, no errors detected)\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3abd48de63dd3956140a30196e0f3dc44998cdba01a694484355c666029b9eeb","failure_pattern":"subscription openshift-nfd/nfd not ready, state=\"\" (OLM still processing, no errors detected)","search_query":"subscription openshift-nfd/nfd not ready, state=\"\" (OLM still processing, no errors detected)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["3abd48de63dd3956140a30196e0f3dc44998cdba01a694484355c666029b9eeb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}