{"meta":{"date":"2026-04-02","timezone":"UTC","generated_at":"2026-08-31T17:12:05Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":6,"failed_runs":5,"runs_with_occurrences":5,"runs_with_failure_pattern_matches":5,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":1,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039846445451841536","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T23:24:25Z"},"failed_at":["e2e"],"failed_test_count":7,"occurrences":[{"row_id":"0ea0ce3309eab72f3e771957ae82aa24336c019d0673ce14a61e39873fa3335b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039846445451841536","occurred_at":"2026-04-02T23:24:25Z","failed_at":"e2e","signature_id":"f62f0cea8a0efc666a80f05794478f17dc1e76b6c132ca2c22eb98b5fbbbd8d1","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:155]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015155c0\u003e: \n failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-wdjndj\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-wdjndj, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-wdjndj\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-wdjndj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00076c300\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-wdjndj\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:155]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015155c0\u003e: \n failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-wdjndj\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-wdjndj, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-wdjndj\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-wdjndj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00076c300\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-wdjndj\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}},{"row_id":"8a9c6510628ef4a73313f0252186543138975be496d970c431e7a9f562d19722","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039846445451841536","occurred_at":"2026-04-02T23:24:25Z","failed_at":"e2e","signature_id":"daa01e0a071bb23981629322b51a8a838a0a20c385d7cccf1bda9ff5fe01bb42","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:132]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015779e0\u003e: \n failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-x5mmq2\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-x5mmq2, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-x5mmq2\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-x5mmq2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a88100\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-x5mmq2\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:132]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015779e0\u003e: \n failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-x5mmq2\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-x5mmq2, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-x5mmq2\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-x5mmq2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a88100\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-x5mmq2\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}},{"row_id":"9cf5f298e0f5525ad5d636d83af90c53d39039d89e3562289b5816ac74160f1f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039846445451841536","occurred_at":"2026-04-02T23:24:25Z","failed_at":"e2e","signature_id":"48700371ca5e3ea6f36170d783d442d058d1cb1424257da18a3fc2d24e97609f","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:103]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00156a3c0\u003e: \n failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-x5nbs9\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-x5nbs9, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-x5nbs9\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-x5nbs9, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000dbddd0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-x5nbs9\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:103]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00156a3c0\u003e: \n failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-x5nbs9\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-x5nbs9, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-x5nbs9\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-x5nbs9, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000dbddd0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-x5nbs9\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}},{"row_id":"9d2687a134958e6be1b4ee409b2a79dddf688db2e4a03d6bbd6d341d734a5f91","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039846445451841536","occurred_at":"2026-04-02T23:24:25Z","failed_at":"e2e","signature_id":"425cb192426cfae26c27c30de57f81c15b90dc678940b4bca02416c7cb2beaed","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:101]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009decc0\u003e: \n failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-782d8g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-782d8g, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-np-labels-taints\\\" in resourcegroup=\\\"rg-np-labels-taints-782d8g\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-782d8g, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000c04df0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-782d8g\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:101]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009decc0\u003e: \n failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-782d8g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-782d8g, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-np-labels-taints\\\" in resourcegroup=\\\"rg-np-labels-taints-782d8g\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-782d8g, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000c04df0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-782d8g\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}},{"row_id":"a170db2935fc79d55777ecb435118037340c803c6f37d8b048e3b21d17a18734","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039846445451841536","occurred_at":"2026-04-02T23:24:25Z","failed_at":"e2e","signature_id":"925d5b5e8d1cb7062ad7f97148097b4bbbd90c0a007478dcd253302060efce42","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011e3890\u003e: \n failed waiting for hcpCluster=\"cluster-stable-4-20-vtmlht\" in resourcegroup=\"rg-stable-4-20-jdtn2v\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-vtmlht in resource group rg-stable-4-20-jdtn2v, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-stable-4-20-vtmlht\\\" in resourcegroup=\\\"rg-stable-4-20-jdtn2v\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-vtmlht in resource group rg-stable-4-20-jdtn2v, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013502e0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-vtmlht in resource group rg-stable-4-20-jdtn2v\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011e3890\u003e: \n failed waiting for hcpCluster=\"cluster-stable-4-20-vtmlht\" in resourcegroup=\"rg-stable-4-20-jdtn2v\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-vtmlht in resource group rg-stable-4-20-jdtn2v, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-stable-4-20-vtmlht\\\" in resourcegroup=\\\"rg-stable-4-20-jdtn2v\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-vtmlht in resource group rg-stable-4-20-jdtn2v, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013502e0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-vtmlht in resource group rg-stable-4-20-jdtn2v\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}},{"row_id":"d42f9c03b37ebe57f114ede6daf32c882346d7a9ed097094ef364865d97072ba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039846445451841536","occurred_at":"2026-04-02T23:24:25Z","failed_at":"e2e","signature_id":"abbe9ccf77f58b9dcecdebc516030b33899c9d00e6c69935ca042cc483f352bf","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:245]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001065020\u003e: \n failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-x8t6s2\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-x8t6s2, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-x8t6s2\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-x8t6s2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0005f7800\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-x8t6s2\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:245]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001065020\u003e: \n failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-x8t6s2\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-x8t6s2, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-x8t6s2\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-x8t6s2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0005f7800\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-x8t6s2\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}},{"row_id":"e02578384602e6abe9f01596b9a179649775e0749f014d48170df4dee41eebac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039846445451841536","occurred_at":"2026-04-02T23:24:25Z","failed_at":"e2e","signature_id":"70daa82f0e74a2a68d0f694e0626bdd2ee7f59b84ad17694935a98cf2693387c","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:105]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00071ba70\u003e: \n failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dmzpzs\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-dmzpzs, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-dmzpzs\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-dmzpzs, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009c5c60\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-dmzpzs\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:105]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00071ba70\u003e: \n failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dmzpzs\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-dmzpzs, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-dmzpzs\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-dmzpzs, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009c5c60\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-dmzpzs\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":7,"failure_pattern_ids":["61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b"],"matched_occurrences":7,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T20:40:02Z"},"failed_at":["e2e"],"failed_test_count":14,"occurrences":[{"row_id":"0ce73f6104664d7ec9557a1d0be168de2496f3556537348504c79ccd85f0c578","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","failed_at":"e2e","signature_id":"4c6fe9b34a98463c74e064da1997e5316b414aefa31a08707da1116847153d72","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:245]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e86750\u003e: \n failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-c7npm9\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-c7npm9, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-c7npm9\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-c7npm9, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0003412b0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-c7npm9\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:245]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e86750\u003e: \n failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-c7npm9\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-c7npm9, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-c7npm9\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-c7npm9, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0003412b0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster oidc-wi-cluster in resource group oidc-wi-c7npm9\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}},{"row_id":"0f350c0aa194f96c1e1aa94fe66144100cd38dd39d36c327186dde15a5f416d7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","failed_at":"e2e","signature_id":"96e939a1dc8a1464c8152e2dcfbf00623eaef4723e51466619b88ddbd5291584","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0008caba0\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-rcgc8p\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-rcgc8p, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-rcgc8p\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-rcgc8p, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00097eb20\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-rcgc8p\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0008caba0\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-rcgc8p\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-rcgc8p, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-rcgc8p\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-rcgc8p, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00097eb20\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-rcgc8p\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}},{"row_id":"67d59a229a1c5a33df1efe2f6cd14ec84e2120ac3112370d3661d362d3b59250","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","failed_at":"e2e","signature_id":"c6f66b20a06674c7be9ba89a2d96b7d1acc8a1ac27f90ece2f2477e63df258d6","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:126]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001784cf0\u003e: \n failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-q97v42\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-q97v42, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-q97v42\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-q97v42, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00095a880\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-q97v42\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:126]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001784cf0\u003e: \n failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-q97v42\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-q97v42, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-q97v42\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-q97v42, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00095a880\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-q97v42\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}},{"row_id":"6b8365184e1f22392bde1ec4baee10e7a8054c761e31f499d56175a7bb88d8e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","failed_at":"e2e","signature_id":"3a274d8c90c6db3b46a4ec71e46a64843c4377cb4ead33281af23bd84b9ab97d","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011f2b40\u003e: \n failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-rpmtxr\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-rpmtxr, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-rpmtxr\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-rpmtxr, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0008cf150\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-rpmtxr\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011f2b40\u003e: \n failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-rpmtxr\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-rpmtxr, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-rpmtxr\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-rpmtxr, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0008cf150\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-rpmtxr\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}},{"row_id":"7c9e0b02739a52de1c3ba1f3c7b104e5125b496786a3c6b9a123e5ae64baa26d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","failed_at":"e2e","signature_id":"1c8fe61fbd43db9169aaa7fbf69609a8728f6d8509220f9a92d6f861ef91f8c0","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00160aa20\u003e: \n failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-sbvxnd\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-sbvxnd, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-sbvxnd\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-sbvxnd, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0003b9b70\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-sbvxnd\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00160aa20\u003e: \n failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-sbvxnd\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-sbvxnd, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-sbvxnd\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-sbvxnd, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0003b9b70\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-sbvxnd\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}},{"row_id":"82d72803a51c8f37ae10c1da56fc146cdafee0eb8e5293e4d137f63b98b5e5a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","failed_at":"e2e","signature_id":"87f712f0082578aceb2139869be1b2173936efbf987880da72d2af8ef95f3557","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:101]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00178a840\u003e: \n failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-kh6vnx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-kh6vnx, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-np-labels-taints\\\" in resourcegroup=\\\"rg-np-labels-taints-kh6vnx\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-kh6vnx, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000670d20\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-kh6vnx\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:101]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00178a840\u003e: \n failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-kh6vnx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-kh6vnx, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-np-labels-taints\\\" in resourcegroup=\\\"rg-np-labels-taints-kh6vnx\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-kh6vnx, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000670d20\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-kh6vnx\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}},{"row_id":"8b2090d674e67628e398f4224c2c84345e8a9a2fc7825a66155c7d277059daf1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","failed_at":"e2e","signature_id":"61f89257fbd679c1f5a3f14ecb2cfdc110c59434b5cb1139ebf64baa58624853","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00103eb70\u003e: \n failed waiting for hcpCluster=\"cluster-stable-4-20-fbtnrz\" in resourcegroup=\"rg-stable-4-20-bcpmj9\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-fbtnrz in resource group rg-stable-4-20-bcpmj9, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-stable-4-20-fbtnrz\\\" in resourcegroup=\\\"rg-stable-4-20-bcpmj9\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-fbtnrz in resource group rg-stable-4-20-bcpmj9, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a221e0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-fbtnrz in resource group rg-stable-4-20-bcpmj9\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00103eb70\u003e: \n failed waiting for hcpCluster=\"cluster-stable-4-20-fbtnrz\" in resourcegroup=\"rg-stable-4-20-bcpmj9\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-fbtnrz in resource group rg-stable-4-20-bcpmj9, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-stable-4-20-fbtnrz\\\" in resourcegroup=\\\"rg-stable-4-20-bcpmj9\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-fbtnrz in resource group rg-stable-4-20-bcpmj9, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a221e0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-fbtnrz in resource group rg-stable-4-20-bcpmj9\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}},{"row_id":"8f5e61535e4b7ca21c62c7cd251c68972704b56058602f49e691b210353347fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","failed_at":"e2e","signature_id":"473710e31048fab0665756ba98c3a4020c0308701ea4c9a9babf37c2b2ab4f39","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:132]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001208660\u003e: \n failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-jxthrg\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-jxthrg, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-jxthrg\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-jxthrg, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001486620\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-jxthrg\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:132]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001208660\u003e: \n failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-jxthrg\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-jxthrg, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-jxthrg\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-jxthrg, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001486620\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster idms-e2e-hcp-cluster in resource group idms-jxthrg\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}},{"row_id":"9feee6de7f79988a47627443c309921db8fb1e202b8c55c4178eec5e4a71718c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","failed_at":"e2e","signature_id":"f3edf87053a3ddfa94c5d07ba8914a620291e1ca95012ba77d27456596d7a2f7","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:127]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009612f0\u003e: \n failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-zccvrs\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster negative-tests-cluster in resource group rg-negative-tests-zccvrs, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-zccvrs\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster negative-tests-cluster in resource group rg-negative-tests-zccvrs, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013b52e0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster negative-tests-cluster in resource group rg-negative-tests-zccvrs\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:127]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009612f0\u003e: \n failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-zccvrs\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster negative-tests-cluster in resource group rg-negative-tests-zccvrs, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"negative-tests-cluster\\\" in resourcegroup=\\\"rg-negative-tests-zccvrs\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster negative-tests-cluster in resource group rg-negative-tests-zccvrs, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013b52e0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster negative-tests-cluster in resource group rg-negative-tests-zccvrs\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}},{"row_id":"ce51519b0701ab911964bf4ee463c416bb29559975800b21f8519078c7abf2ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","failed_at":"e2e","signature_id":"a77aba6e2f5896ef8dd63e5789b9c1381fb9c0ef40adec0f73550907fb8efca7","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012cc420\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-21-zfhpmw\" in resourcegroup=\"rg-zstream-upgrade-4-21-lzhvk4\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-zfhpmw in resource group rg-zstream-upgrade-4-21-lzhvk4, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-zfhpmw\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-lzhvk4\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-zfhpmw in resource group rg-zstream-upgrade-4-21-lzhvk4, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00153ccf0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-zfhpmw in resource group rg-zstream-upgrade-4-21-lzhvk4\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012cc420\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-21-zfhpmw\" in resourcegroup=\"rg-zstream-upgrade-4-21-lzhvk4\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-zfhpmw in resource group rg-zstream-upgrade-4-21-lzhvk4, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-zfhpmw\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-lzhvk4\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-zfhpmw in resource group rg-zstream-upgrade-4-21-lzhvk4, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00153ccf0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-zfhpmw in resource group rg-zstream-upgrade-4-21-lzhvk4\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}},{"row_id":"ddd07d751395c086c610cdb2d6b669eec6ffe99b85c3c05b9a93ca6fab5e3dae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","failed_at":"e2e","signature_id":"bd8c64743544c200535996997a509a9d5a3ba8e598a513d39001e796b305da6a","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0016a0f60\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-9sv9gw\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-g6kp9g\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000986060\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0016a0f60\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-9sv9gw\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-g6kp9g\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000986060\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}},{"row_id":"deb92528feb402acdd7912e403ba317781cc6831215eabf566b31a9ab390afb2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","failed_at":"e2e","signature_id":"6b5479644ee17536d3c82e89ebbb26e7571bb914b168ab9d74f281baf41fd90c","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:105]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a3c3c0\u003e: \n failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-cw7dq4\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-cw7dq4, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-cw7dq4\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-cw7dq4, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009e6fa0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-cw7dq4\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:105]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a3c3c0\u003e: \n failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-cw7dq4\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-cw7dq4, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-cw7dq4\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-cw7dq4, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009e6fa0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-cw7dq4\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}},{"row_id":"e519f946b792e0b495b784b4010803a5c515089a15922409fd0b5286057f69db","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","failed_at":"e2e","signature_id":"807f4398a1b973a767b11bdff3ba6ffe1f98c0444aa1aecbf515704d79fa35f2","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015b4e40\u003e: \n failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-b5bsvg\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-b5bsvg, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-b5bsvg\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-b5bsvg, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0000d3c50\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-b5bsvg\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015b4e40\u003e: \n failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-b5bsvg\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-b5bsvg, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-b5bsvg\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-b5bsvg, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0000d3c50\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-b5bsvg\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}},{"row_id":"eb7df46babaa8ee3d33105d3f51528ec681d555fdd4c7f63bef241b5a1ce5716","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","failed_at":"e2e","signature_id":"5e59c002f97b94a2b5076e852a21ebe711b44b0942e4ecc5862f23348be56977","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:103]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00071e810\u003e: \n failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-wmxhsh\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-wmxhsh, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-wmxhsh\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-wmxhsh, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fd5a50\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-wmxhsh\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:103]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00071e810\u003e: \n failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-wmxhsh\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-wmxhsh, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-wmxhsh\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-wmxhsh, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fd5a50\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-wmxhsh\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":14,"failure_pattern_ids":["61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b"],"matched_occurrences":14,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039753351280726016","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T17:14:28Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"0a15347984c7f65d34b3d0d5e0338d36be9b7dec79eea4a622254c5a25c4d6ce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039753351280726016","occurred_at":"2026-04-02T17:14:28Z","failed_at":"e2e","signature_id":"355229726f681e41eedcc021a5f6f4da31e67f116c3a8985a272f805822bcf87","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008d7580\u003e: \n failed to create HCP cluster cluster-zstream-4-19-456qg5: failed waiting for cluster=\"cluster-zstream-4-19-456qg5\" in resourcegroup=\"rg-zstream-upgrade-4-19-p762zb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\",\n \"name\": \"78ee839b-27e8-45ab-9bef-ab361680e06a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T17:27:46.835065402Z\",\n \"endTime\": \"2026-04-02T17:27:57.932415489Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-456qg5: failed waiting for cluster=\\\"cluster-zstream-4-19-456qg5\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-p762zb\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\\\",\\n \\\"name\\\": \\\"78ee839b-27e8-45ab-9bef-ab361680e06a\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T17:27:46.835065402Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T17:27:57.932415489Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0008d7560\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-19-456qg5\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-p762zb\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\\\",\\n \\\"name\\\": \\\"78ee839b-27e8-45ab-9bef-ab361680e06a\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T17:27:46.835065402Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T17:27:57.932415489Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001317380\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"499\"],\n \"X-Ms-Request-Id\": [\n \"f6c45648-41ff-430b-8947-420c54510b28\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T172807Z:e6630a99-063f-431a-9926-65553b1b3548\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/26acb1f6-ebb4-4c9f-9e33-8c891414cd26\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n \"X-Msedge-Ref\": [\n \"Ref A: C88F82DD8BF54F1BA57C9FED21B63FDF Ref B: CHI261140514034 Ref C: 2026-04-02T17:28:07Z\",\n ],\n \"Date\": [\n \"Thu, 02 Apr 2026 17:28:07 GMT\",\n ],\n...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008d7580\u003e: \n failed to create HCP cluster cluster-zstream-4-19-456qg5: failed waiting for cluster=\"cluster-zstream-4-19-456qg5\" in resourcegroup=\"rg-zstream-upgrade-4-19-p762zb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\",\n \"name\": \"78ee839b-27e8-45ab-9bef-ab361680e06a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T17:27:46.835065402Z\",\n \"endTime\": \"2026-04-02T17:27:57.932415489Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-456qg5: failed waiting for cluster=\\\"cluster-zstream-4-19-456qg5\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-p762zb\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\\\",\\n \\\"name\\\": \\\"78ee839b-27e8-45ab-9bef-ab361680e06a\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T17:27:46.835065402Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T17:27:57.932415489Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0008d7560\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-19-456qg5\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-p762zb\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\\\",\\n \\\"name\\\": \\\"78ee839b-27e8-45ab-9bef-ab361680e06a\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T17:27:46.835065402Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T17:27:57.932415489Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001317380\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"499\"],\n \"X-Ms-Request-Id\": [\n \"f6c45648-41ff-430b-8947-420c54510b28\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T172807Z:e6630a99-063f-431a-9926-65553b1b3548\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/26acb1f6-ebb4-4c9f-9e33-8c891414cd26\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n \"X-Msedge-Ref\": [\n \"Ref A: C88F82DD8BF54F1BA57C9FED21B63FDF Ref B: CHI261140514034 Ref C: 2026-04-02T17:28:07Z\",\n ],\n \"Date\": [\n \"Thu, 02 Apr 2026 17:28:07 GMT\",\n ],\n...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9d240f3596a0d7016c609b7924c0509e3c0ca3a68bbc6af54e7ce19b76a8b991","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5a39016a623d7e129217389e275e891370307edc424fc5133c93ec6e3c2d127a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039753351280726016","occurred_at":"2026-04-02T17:14:28Z","failed_at":"e2e","signature_id":"5555990c760129d0559f6b21d12a328fcac98cdae5cc03808dcd0781d1280bf1","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0004066e0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-b8fnr6: failed waiting for cluster=\"np-version-upgrade-cluster-b8fnr6\" in resourcegroup=\"rg-np-version-upgrade-b8fnr6-jjtft5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/32d8b3c7-04b2-4a25-9b60-ad0cd12fd014\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/32d8b3c7-04b2-4a25-9b60-ad0cd12fd014\",\n \"name\": \"32d8b3c7-04b2-4a25-9b60-ad0cd12fd014\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T17:22:57.156170095Z\",\n \"endTime\": \"2026-04-02T17:23:13.582154512Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-version-upgrade-cluster-b8fnr6: failed waiting for cluster=\\\"np-version-upgrade-cluster-b8fnr6\\\" in resourcegroup=\\\"rg-np-version-upgrade-b8fnr6-jjtft5\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/32d8b3c7-04b2-4a25-9b60-ad0cd12fd014\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/32d8b3c7-04b2-4a25-9b60-ad0cd12fd014\\\",\\n \\\"name\\\": \\\"32d8b3c7-04b2-4a25-9b60-ad0cd12fd014\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T17:22:57.156170095Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T17:23:13.582154512Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0004066c0\u003e{\n msg: \"failed waiting for cluster=\\\"np-version-upgrade-cluster-b8fnr6\\\" in resourcegroup=\\\"rg-np-version-upgrade-b8fnr6-jjtft5\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/32d8b3c7-04b2-4a25-9b60-ad0cd12fd014\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/32d8b3c7-04b2-4a25-9b60-ad0cd12fd014\\\",\\n \\\"name\\\": \\\"32d8b3c7-04b2-4a25-9b60-ad0cd12fd014\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T17:22:57.156170095Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T17:23:13.582154512Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001438780\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"a49f00f4-104e-4fc7-8144-4b711f384af9\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 0EF81C0355DE4A3EBDA3CF679685BB01 Ref B: CH1AA2020617023 Ref C: 2026-04-02T17:23:18Z\",\n ],\n \"Expires\": [\"-1\"],\n \"X-Ms-Request-Id\": [\n \"fa648e5b-e803-4818-89e0-4eeb3ec28e06\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n \"Pragma\": [\"no-cache\"],\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0004066e0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-b8fnr6: failed waiting for cluster=\"np-version-upgrade-cluster-b8fnr6\" in resourcegroup=\"rg-np-version-upgrade-b8fnr6-jjtft5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/32d8b3c7-04b2-4a25-9b60-ad0cd12fd014\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/32d8b3c7-04b2-4a25-9b60-ad0cd12fd014\",\n \"name\": \"32d8b3c7-04b2-4a25-9b60-ad0cd12fd014\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T17:22:57.156170095Z\",\n \"endTime\": \"2026-04-02T17:23:13.582154512Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-version-upgrade-cluster-b8fnr6: failed waiting for cluster=\\\"np-version-upgrade-cluster-b8fnr6\\\" in resourcegroup=\\\"rg-np-version-upgrade-b8fnr6-jjtft5\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/32d8b3c7-04b2-4a25-9b60-ad0cd12fd014\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/32d8b3c7-04b2-4a25-9b60-ad0cd12fd014\\\",\\n \\\"name\\\": \\\"32d8b3c7-04b2-4a25-9b60-ad0cd12fd014\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T17:22:57.156170095Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T17:23:13.582154512Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0004066c0\u003e{\n msg: \"failed waiting for cluster=\\\"np-version-upgrade-cluster-b8fnr6\\\" in resourcegroup=\\\"rg-np-version-upgrade-b8fnr6-jjtft5\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/32d8b3c7-04b2-4a25-9b60-ad0cd12fd014\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/32d8b3c7-04b2-4a25-9b60-ad0cd12fd014\\\",\\n \\\"name\\\": \\\"32d8b3c7-04b2-4a25-9b60-ad0cd12fd014\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T17:22:57.156170095Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T17:23:13.582154512Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001438780\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"a49f00f4-104e-4fc7-8144-4b711f384af9\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 0EF81C0355DE4A3EBDA3CF679685BB01 Ref B: CH1AA2020617023 Ref C: 2026-04-02T17:23:18Z\",\n ],\n \"Expires\": [\"-1\"],\n \"X-Ms-Request-Id\": [\n \"fa648e5b-e803-4818-89e0-4eeb3ec28e06\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n \"Pragma\": [\"no-cache\"],\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9d240f3596a0d7016c609b7924c0509e3c0ca3a68bbc6af54e7ce19b76a8b991","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6a7e34c127c39397c3d5d0087c384c54053b42f32f63c5f0c07b034e405a3481","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039753351280726016","occurred_at":"2026-04-02T17:14:28Z","failed_at":"e2e","signature_id":"9b65bed4ba440d31a2069975341012840e9a50435e2711db9b3bacbab843329f","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0014ded40\u003e: \n failed to create HCP cluster cluster-zstream-4-21-x57jw2: failed waiting for cluster=\"cluster-zstream-4-21-x57jw2\" in resourcegroup=\"rg-zstream-upgrade-4-21-7ld9jr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/706f031f-b30d-4537-9779-a53a5765e7d7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/706f031f-b30d-4537-9779-a53a5765e7d7\",\n \"name\": \"706f031f-b30d-4537-9779-a53a5765e7d7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T17:28:00.969777194Z\",\n \"endTime\": \"2026-04-02T17:28:53.840813339Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-21-x57jw2: failed waiting for cluster=\\\"cluster-zstream-4-21-x57jw2\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-7ld9jr\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/706f031f-b30d-4537-9779-a53a5765e7d7\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/706f031f-b30d-4537-9779-a53a5765e7d7\\\",\\n \\\"name\\\": \\\"706f031f-b30d-4537-9779-a53a5765e7d7\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T17:28:00.969777194Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T17:28:53.840813339Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0014ded20\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-x57jw2\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-7ld9jr\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/706f031f-b30d-4537-9779-a53a5765e7d7\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/706f031f-b30d-4537-9779-a53a5765e7d7\\\",\\n \\\"name\\\": \\\"706f031f-b30d-4537-9779-a53a5765e7d7\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T17:28:00.969777194Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T17:28:53.840813339Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0014cdec0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Expires\": [\"-1\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"95dbdfdf-eccc-4906-8280-98e36991764d\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Msedge-Ref\": [\n \"Ref A: A7B02FC3CF694D2B911CFE7230B877F6 Ref B: CH1AA2020617011 Ref C: 2026-04-02T17:29:02Z\",\n ],\n \"Date\": [\n \"Thu, 02 Apr 2026 17:29:02 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T172902Z:95dbdfdf-eccc-4906-8280-98e36991764d\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=93...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0014ded40\u003e: \n failed to create HCP cluster cluster-zstream-4-21-x57jw2: failed waiting for cluster=\"cluster-zstream-4-21-x57jw2\" in resourcegroup=\"rg-zstream-upgrade-4-21-7ld9jr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/706f031f-b30d-4537-9779-a53a5765e7d7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/706f031f-b30d-4537-9779-a53a5765e7d7\",\n \"name\": \"706f031f-b30d-4537-9779-a53a5765e7d7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T17:28:00.969777194Z\",\n \"endTime\": \"2026-04-02T17:28:53.840813339Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-21-x57jw2: failed waiting for cluster=\\\"cluster-zstream-4-21-x57jw2\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-7ld9jr\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/706f031f-b30d-4537-9779-a53a5765e7d7\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/706f031f-b30d-4537-9779-a53a5765e7d7\\\",\\n \\\"name\\\": \\\"706f031f-b30d-4537-9779-a53a5765e7d7\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T17:28:00.969777194Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T17:28:53.840813339Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0014ded20\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-x57jw2\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-7ld9jr\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/706f031f-b30d-4537-9779-a53a5765e7d7\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/706f031f-b30d-4537-9779-a53a5765e7d7\\\",\\n \\\"name\\\": \\\"706f031f-b30d-4537-9779-a53a5765e7d7\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T17:28:00.969777194Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T17:28:53.840813339Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0014cdec0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Expires\": [\"-1\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"95dbdfdf-eccc-4906-8280-98e36991764d\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Msedge-Ref\": [\n \"Ref A: A7B02FC3CF694D2B911CFE7230B877F6 Ref B: CH1AA2020617011 Ref C: 2026-04-02T17:29:02Z\",\n ],\n \"Date\": [\n \"Thu, 02 Apr 2026 17:29:02 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T172902Z:95dbdfdf-eccc-4906-8280-98e36991764d\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Envoy-Upstream-Service-Time\": [\"10\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=93...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9d240f3596a0d7016c609b7924c0509e3c0ca3a68bbc6af54e7ce19b76a8b991","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a75ec0439e6c871a195fd774a95873c4774f3020f7775b50569bf62a2839b8b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039753351280726016","occurred_at":"2026-04-02T17:14:28Z","failed_at":"e2e","signature_id":"42f9dbe161009cc54c6fd91be6ef6725fe5af524a9c492cbb4f914f791198d53","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0005873e0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-qhj6bm: failed waiting for cluster=\"np-version-upgrade-cluster-qhj6bm\" in resourcegroup=\"rg-np-version-upgrade-qhj6bm-hp2rjl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\",\n \"name\": \"774a88cb-0063-4a09-b407-ee6a7f59a9e6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T17:22:52.081012471Z\",\n \"endTime\": \"2026-04-02T17:23:13.48665448Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-version-upgrade-cluster-qhj6bm: failed waiting for cluster=\\\"np-version-upgrade-cluster-qhj6bm\\\" in resourcegroup=\\\"rg-np-version-upgrade-qhj6bm-hp2rjl\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\\\",\\n \\\"name\\\": \\\"774a88cb-0063-4a09-b407-ee6a7f59a9e6\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T17:22:52.081012471Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T17:23:13.48665448Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0005873c0\u003e{\n msg: \"failed waiting for cluster=\\\"np-version-upgrade-cluster-qhj6bm\\\" in resourcegroup=\\\"rg-np-version-upgrade-qhj6bm-hp2rjl\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\\\",\\n \\\"name\\\": \\\"774a88cb-0063-4a09-b407-ee6a7f59a9e6\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T17:22:52.081012471Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T17:23:13.48665448Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0010284e0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Length\": [\"498\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"8a74d4be-8a1d-4808-936e-22bae5b4ccd3\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: 37CA783E55BC4DE4BA0F59B834B5F490 Ref B: CHI261140504062 Ref C: 2026-04-02T17:23:22Z\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"X-Envoy-Upstream-Service-Time\": [\"20\"],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T172322Z:8a74d4be-8a1d-4808-936e-22bae5b4ccd3\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"c674f25c-3bae-4b96-9f1e...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0005873e0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-qhj6bm: failed waiting for cluster=\"np-version-upgrade-cluster-qhj6bm\" in resourcegroup=\"rg-np-version-upgrade-qhj6bm-hp2rjl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\",\n \"name\": \"774a88cb-0063-4a09-b407-ee6a7f59a9e6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T17:22:52.081012471Z\",\n \"endTime\": \"2026-04-02T17:23:13.48665448Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-version-upgrade-cluster-qhj6bm: failed waiting for cluster=\\\"np-version-upgrade-cluster-qhj6bm\\\" in resourcegroup=\\\"rg-np-version-upgrade-qhj6bm-hp2rjl\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\\\",\\n \\\"name\\\": \\\"774a88cb-0063-4a09-b407-ee6a7f59a9e6\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T17:22:52.081012471Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T17:23:13.48665448Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0005873c0\u003e{\n msg: \"failed waiting for cluster=\\\"np-version-upgrade-cluster-qhj6bm\\\" in resourcegroup=\\\"rg-np-version-upgrade-qhj6bm-hp2rjl\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\\\",\\n \\\"name\\\": \\\"774a88cb-0063-4a09-b407-ee6a7f59a9e6\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T17:22:52.081012471Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T17:23:13.48665448Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0010284e0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Length\": [\"498\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"8a74d4be-8a1d-4808-936e-22bae5b4ccd3\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: 37CA783E55BC4DE4BA0F59B834B5F490 Ref B: CHI261140504062 Ref C: 2026-04-02T17:23:22Z\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"X-Envoy-Upstream-Service-Time\": [\"20\"],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T172322Z:8a74d4be-8a1d-4808-936e-22bae5b4ccd3\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"c674f25c-3bae-4b96-9f1e...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9d240f3596a0d7016c609b7924c0509e3c0ca3a68bbc6af54e7ce19b76a8b991","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b2b4a784ee9293a01d754a91a9c229a019a446ccab0fc9f70821e24cae6b4d46","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039753351280726016","occurred_at":"2026-04-02T17:14:28Z","failed_at":"e2e","signature_id":"20e46711d3e35710a980843025c80ad4a7b4cde8409fdf3d9c4a64498fb53b05","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000732640\u003e: \n failed to create HCP cluster cluster-zstream-4-20-7mzwzp: failed waiting for cluster=\"cluster-zstream-4-20-7mzwzp\" in resourcegroup=\"rg-zstream-upgrade-4-20-tq69zx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\",\n \"name\": \"e3ee4618-26eb-473e-867f-a23c3644e2bb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T17:31:57.472030714Z\",\n \"endTime\": \"2026-04-02T17:32:55.049606818Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-20-7mzwzp: failed waiting for cluster=\\\"cluster-zstream-4-20-7mzwzp\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tq69zx\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\\\",\\n \\\"name\\\": \\\"e3ee4618-26eb-473e-867f-a23c3644e2bb\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T17:31:57.472030714Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T17:32:55.049606818Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000732600\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-7mzwzp\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tq69zx\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\\\",\\n \\\"name\\\": \\\"e3ee4618-26eb-473e-867f-a23c3644e2bb\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T17:31:57.472030714Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T17:32:55.049606818Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0014af200\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadaeast/81439e1d-c310-4abd-91e5-f37963acb072\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Length\": [\"499\"],\n \"Content-Type\": [\"application/json\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 41853D22B093487EB33229F443FF6553 Ref B: CHI261140513025 Ref C: 2026-04-02T17:33:00Z\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000732640\u003e: \n failed to create HCP cluster cluster-zstream-4-20-7mzwzp: failed waiting for cluster=\"cluster-zstream-4-20-7mzwzp\" in resourcegroup=\"rg-zstream-upgrade-4-20-tq69zx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\",\n \"name\": \"e3ee4618-26eb-473e-867f-a23c3644e2bb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T17:31:57.472030714Z\",\n \"endTime\": \"2026-04-02T17:32:55.049606818Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-20-7mzwzp: failed waiting for cluster=\\\"cluster-zstream-4-20-7mzwzp\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tq69zx\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\\\",\\n \\\"name\\\": \\\"e3ee4618-26eb-473e-867f-a23c3644e2bb\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T17:31:57.472030714Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T17:32:55.049606818Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000732600\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-7mzwzp\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tq69zx\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\\\",\\n \\\"name\\\": \\\"e3ee4618-26eb-473e-867f-a23c3644e2bb\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T17:31:57.472030714Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T17:32:55.049606818Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0014af200\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadaeast/81439e1d-c310-4abd-91e5-f37963acb072\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Length\": [\"499\"],\n \"Content-Type\": [\"application/json\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 41853D22B093487EB33229F443FF6553 Ref B: CHI261140513025 Ref C: 2026-04-02T17:33:00Z\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9d240f3596a0d7016c609b7924c0509e3c0ca3a68bbc6af54e7ce19b76a8b991","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["9d240f3596a0d7016c609b7924c0509e3c0ca3a68bbc6af54e7ce19b76a8b991"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T08:48:43Z"},"failed_at":["e2e"],"failed_test_count":8,"occurrences":[{"row_id":"010bc18a3604de33463092cced6723263c50e7538970877333e0dd7286aebcc2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","occurred_at":"2026-04-02T08:48:43Z","failed_at":"e2e","signature_id":"9a9415a468f807e3f900a81033319d4a094d50d5a9a6a438265cad1fac141e42","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e5d680\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-zlcbwt: failed waiting for cluster=\"np-version-upgrade-cluster-zlcbwt\" in resourcegroup=\"rg-np-version-upgrade-zlcbwt-xdw7n4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f6a47a17-e228-4123-8880-e17806ecd15c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f6a47a17-e228-4123-8880-e17806ecd15c\",\n \"name\": \"f6a47a17-e228-4123-8880-e17806ecd15c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T09:00:13.820056162Z\",\n \"endTime\": \"2026-04-02T09:00:48.215590338Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-version-upgrade-cluster-zlcbwt: failed waiting for cluster=\\\"np-version-upgrade-cluster-zlcbwt\\\" in resourcegroup=\\\"rg-np-version-upgrade-zlcbwt-xdw7n4\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f6a47a17-e228-4123-8880-e17806ecd15c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f6a47a17-e228-4123-8880-e17806ecd15c\\\",\\n \\\"name\\\": \\\"f6a47a17-e228-4123-8880-e17806ecd15c\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T09:00:13.820056162Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T09:00:48.215590338Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e5d660\u003e{\n msg: \"failed waiting for cluster=\\\"np-version-upgrade-cluster-zlcbwt\\\" in resourcegroup=\\\"rg-np-version-upgrade-zlcbwt-xdw7n4\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f6a47a17-e228-4123-8880-e17806ecd15c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f6a47a17-e228-4123-8880-e17806ecd15c\\\",\\n \\\"name\\\": \\\"f6a47a17-e228-4123-8880-e17806ecd15c\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T09:00:13.820056162Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T09:00:48.215590338Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00114aba0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T090055Z:9658587b-26d3-458b-83e1-b589efdf4111\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 09:00:54 GMT\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"9658587b-26d3-458b-83e1-b589efdf4111\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Length\": [\"499\"],\n \"X-Ms-Request-Id\": [\n \"de87e374-c0ba-41f3-b4e8-34237314bf28\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"23\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,obje...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e5d680\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-zlcbwt: failed waiting for cluster=\"np-version-upgrade-cluster-zlcbwt\" in resourcegroup=\"rg-np-version-upgrade-zlcbwt-xdw7n4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f6a47a17-e228-4123-8880-e17806ecd15c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f6a47a17-e228-4123-8880-e17806ecd15c\",\n \"name\": \"f6a47a17-e228-4123-8880-e17806ecd15c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T09:00:13.820056162Z\",\n \"endTime\": \"2026-04-02T09:00:48.215590338Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-version-upgrade-cluster-zlcbwt: failed waiting for cluster=\\\"np-version-upgrade-cluster-zlcbwt\\\" in resourcegroup=\\\"rg-np-version-upgrade-zlcbwt-xdw7n4\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f6a47a17-e228-4123-8880-e17806ecd15c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f6a47a17-e228-4123-8880-e17806ecd15c\\\",\\n \\\"name\\\": \\\"f6a47a17-e228-4123-8880-e17806ecd15c\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T09:00:13.820056162Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T09:00:48.215590338Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e5d660\u003e{\n msg: \"failed waiting for cluster=\\\"np-version-upgrade-cluster-zlcbwt\\\" in resourcegroup=\\\"rg-np-version-upgrade-zlcbwt-xdw7n4\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f6a47a17-e228-4123-8880-e17806ecd15c\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f6a47a17-e228-4123-8880-e17806ecd15c\\\",\\n \\\"name\\\": \\\"f6a47a17-e228-4123-8880-e17806ecd15c\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T09:00:13.820056162Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T09:00:48.215590338Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00114aba0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T090055Z:9658587b-26d3-458b-83e1-b589efdf4111\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 09:00:54 GMT\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"9658587b-26d3-458b-83e1-b589efdf4111\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Length\": [\"499\"],\n \"X-Ms-Request-Id\": [\n \"de87e374-c0ba-41f3-b4e8-34237314bf28\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"23\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,obje...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9d240f3596a0d7016c609b7924c0509e3c0ca3a68bbc6af54e7ce19b76a8b991","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"25233af0f55ca8cfacfa6b5d3c8c89020903cae050fcef73aaa5dc4ce297ec53","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","occurred_at":"2026-04-02T08:48:43Z","failed_at":"e2e","signature_id":"e12a845dc4147e276bce346cf5007864692316e60bb487e5d88e3368003a798a","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000d04420\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-fd27fq: failed waiting for cluster=\"np-version-upgrade-cluster-fd27fq\" in resourcegroup=\"rg-np-version-upgrade-fd27fq-kmzsnf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/faaed912-8c9c-4b9c-a560-ae5d271fe446\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/faaed912-8c9c-4b9c-a560-ae5d271fe446\",\n \"name\": \"faaed912-8c9c-4b9c-a560-ae5d271fe446\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T09:03:17.642456763Z\",\n \"endTime\": \"2026-04-02T09:03:33.609008884Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-version-upgrade-cluster-fd27fq: failed waiting for cluster=\\\"np-version-upgrade-cluster-fd27fq\\\" in resourcegroup=\\\"rg-np-version-upgrade-fd27fq-kmzsnf\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/faaed912-8c9c-4b9c-a560-ae5d271fe446\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/faaed912-8c9c-4b9c-a560-ae5d271fe446\\\",\\n \\\"name\\\": \\\"faaed912-8c9c-4b9c-a560-ae5d271fe446\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T09:03:17.642456763Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T09:03:33.609008884Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000d04400\u003e{\n msg: \"failed waiting for cluster=\\\"np-version-upgrade-cluster-fd27fq\\\" in resourcegroup=\\\"rg-np-version-upgrade-fd27fq-kmzsnf\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/faaed912-8c9c-4b9c-a560-ae5d271fe446\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/faaed912-8c9c-4b9c-a560-ae5d271fe446\\\",\\n \\\"name\\\": \\\"faaed912-8c9c-4b9c-a560-ae5d271fe446\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T09:03:17.642456763Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T09:03:33.609008884Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00142c780\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"499\"],\n \"Content-Type\": [\"application/json\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Request-Id\": [\n \"22fdfd66-2ef7-48dc-92f9-15957d9814a7\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADAEAST:20260402T090338Z:b2299bf9-2580-4e04-bcf6-90257fc5e438\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadaeast/bc50bcf7-9fe7-40b0-8517-584011411f7c\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"b2299bf9-2580-4e04-bcf6-90257fc5e438\",\n ],\n \"Strict-Transport-Security\": [\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000d04420\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-fd27fq: failed waiting for cluster=\"np-version-upgrade-cluster-fd27fq\" in resourcegroup=\"rg-np-version-upgrade-fd27fq-kmzsnf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/faaed912-8c9c-4b9c-a560-ae5d271fe446\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/faaed912-8c9c-4b9c-a560-ae5d271fe446\",\n \"name\": \"faaed912-8c9c-4b9c-a560-ae5d271fe446\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T09:03:17.642456763Z\",\n \"endTime\": \"2026-04-02T09:03:33.609008884Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-version-upgrade-cluster-fd27fq: failed waiting for cluster=\\\"np-version-upgrade-cluster-fd27fq\\\" in resourcegroup=\\\"rg-np-version-upgrade-fd27fq-kmzsnf\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/faaed912-8c9c-4b9c-a560-ae5d271fe446\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/faaed912-8c9c-4b9c-a560-ae5d271fe446\\\",\\n \\\"name\\\": \\\"faaed912-8c9c-4b9c-a560-ae5d271fe446\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T09:03:17.642456763Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T09:03:33.609008884Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000d04400\u003e{\n msg: \"failed waiting for cluster=\\\"np-version-upgrade-cluster-fd27fq\\\" in resourcegroup=\\\"rg-np-version-upgrade-fd27fq-kmzsnf\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/faaed912-8c9c-4b9c-a560-ae5d271fe446\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/faaed912-8c9c-4b9c-a560-ae5d271fe446\\\",\\n \\\"name\\\": \\\"faaed912-8c9c-4b9c-a560-ae5d271fe446\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T09:03:17.642456763Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T09:03:33.609008884Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00142c780\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"499\"],\n \"Content-Type\": [\"application/json\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Request-Id\": [\n \"22fdfd66-2ef7-48dc-92f9-15957d9814a7\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADAEAST:20260402T090338Z:b2299bf9-2580-4e04-bcf6-90257fc5e438\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadaeast/bc50bcf7-9fe7-40b0-8517-584011411f7c\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"b2299bf9-2580-4e04-bcf6-90257fc5e438\",\n ],\n \"Strict-Transport-Security\": [\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9d240f3596a0d7016c609b7924c0509e3c0ca3a68bbc6af54e7ce19b76a8b991","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2d7c144366c1af22daff22ce57d051c333bd617604399575264ceb9967dc204a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","occurred_at":"2026-04-02T08:48:43Z","failed_at":"e2e","signature_id":"fd76359852bfe92722186a57e697c3c69f783a8157d4dcae452c9c7c11c50934","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000880260\u003e: \n failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-6tff8z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-6tff8z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: ServiceUnavailable\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ServiceUnavailable\",\n \"message\": \"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster basic-cluster: failed starting cluster creation \\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-6tff8z\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-6tff8z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"ServiceUnavailable\\\",\\n \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000880200\u003e{\n msg: \"failed starting cluster creation \\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-6tff8z\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-6tff8z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"ServiceUnavailable\\\",\\n \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012ca1b0\u003e{\n ErrorCode: \"ServiceUnavailable\",\n StatusCode: 503,\n RawResponse: {\n Status: \"503 Service Unavailable\",\n StatusCode: 503,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Request-Id\": [\n \"2a87bdad-8340-4cda-a740-2d7cc278c733\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"294bd1e5-7abf-4f05-a3c2-931fa8a72c4e\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260402T085750Z:294bd1e5-7abf-4f05-a3c2-931fa8a72c4e\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Date\": [\n \"Thu, 02 Apr 2026 08:57:50 GMT\",\n ],\n \"Content-Type\": [\"application/json\"],\n \"X-Envoy-Upstream-Service-Time\": [\"2649\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/cc9e8570-fcc2-4ba9-9d49-08bb67825537\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Content-Length\": [\"257\"],\n \"Retry-After\": [\"59\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Fail...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000880260\u003e: \n failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-6tff8z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-6tff8z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: ServiceUnavailable\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ServiceUnavailable\",\n \"message\": \"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster basic-cluster: failed starting cluster creation \\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-6tff8z\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-6tff8z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"ServiceUnavailable\\\",\\n \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000880200\u003e{\n msg: \"failed starting cluster creation \\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-6tff8z\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-6tff8z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"ServiceUnavailable\\\",\\n \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012ca1b0\u003e{\n ErrorCode: \"ServiceUnavailable\",\n StatusCode: 503,\n RawResponse: {\n Status: \"503 Service Unavailable\",\n StatusCode: 503,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Request-Id\": [\n \"2a87bdad-8340-4cda-a740-2d7cc278c733\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"294bd1e5-7abf-4f05-a3c2-931fa8a72c4e\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260402T085750Z:294bd1e5-7abf-4f05-a3c2-931fa8a72c4e\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Date\": [\n \"Thu, 02 Apr 2026 08:57:50 GMT\",\n ],\n \"Content-Type\": [\"application/json\"],\n \"X-Envoy-Upstream-Service-Time\": [\"2649\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/cc9e8570-fcc2-4ba9-9d49-08bb67825537\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Content-Length\": [\"257\"],\n \"Retry-After\": [\"59\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Fail...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"47c4fe17c8b40b438461015820e355e731ca2f90428cea3ac600911bb9a03df7","failure_pattern":"ERROR CODE: ServiceUnavailable; detail message Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ServiceUnavailable"}},{"row_id":"38a6041fe1932c4cd6e229abb692a320ea848247db9a668fcb8a0db557113b4f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","occurred_at":"2026-04-02T08:48:43Z","failed_at":"e2e","signature_id":"69b22aaa698b3b1760bc99ff5999083f5c0c44c4dcf517eb76bf27f757a7e495","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a2f2c0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-b88v27: failed waiting for cluster=\"cluster-zstream-4-20-b88v27\" in resourcegroup=\"rg-zstream-upgrade-4-20-lmrfzn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e3a763-ff68-4aa2-acc4-15b05e869923\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e3a763-ff68-4aa2-acc4-15b05e869923\",\n \"name\": \"d0e3a763-ff68-4aa2-acc4-15b05e869923\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T09:02:34.406522828Z\",\n \"endTime\": \"2026-04-02T09:02:43.014191999Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-20-b88v27: failed waiting for cluster=\\\"cluster-zstream-4-20-b88v27\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-lmrfzn\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e3a763-ff68-4aa2-acc4-15b05e869923\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e3a763-ff68-4aa2-acc4-15b05e869923\\\",\\n \\\"name\\\": \\\"d0e3a763-ff68-4aa2-acc4-15b05e869923\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T09:02:34.406522828Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T09:02:43.014191999Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a2f2a0\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-b88v27\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-lmrfzn\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e3a763-ff68-4aa2-acc4-15b05e869923\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e3a763-ff68-4aa2-acc4-15b05e869923\\\",\\n \\\"name\\\": \\\"d0e3a763-ff68-4aa2-acc4-15b05e869923\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T09:02:34.406522828Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T09:02:43.014191999Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0008428a0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Pragma\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADAEAST:20260402T090245Z:f15f4ae6-036f-4104-9d3c-5b007bb48b61\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Length\": [\"499\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadaeast/b395eb27-1a6a-4aac-b717-f00f205f4a7c\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n \"X-Ms-Correlation-Request-Id\": [\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a2f2c0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-b88v27: failed waiting for cluster=\"cluster-zstream-4-20-b88v27\" in resourcegroup=\"rg-zstream-upgrade-4-20-lmrfzn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e3a763-ff68-4aa2-acc4-15b05e869923\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e3a763-ff68-4aa2-acc4-15b05e869923\",\n \"name\": \"d0e3a763-ff68-4aa2-acc4-15b05e869923\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T09:02:34.406522828Z\",\n \"endTime\": \"2026-04-02T09:02:43.014191999Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-20-b88v27: failed waiting for cluster=\\\"cluster-zstream-4-20-b88v27\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-lmrfzn\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e3a763-ff68-4aa2-acc4-15b05e869923\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e3a763-ff68-4aa2-acc4-15b05e869923\\\",\\n \\\"name\\\": \\\"d0e3a763-ff68-4aa2-acc4-15b05e869923\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T09:02:34.406522828Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T09:02:43.014191999Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a2f2a0\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-b88v27\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-lmrfzn\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e3a763-ff68-4aa2-acc4-15b05e869923\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d0e3a763-ff68-4aa2-acc4-15b05e869923\\\",\\n \\\"name\\\": \\\"d0e3a763-ff68-4aa2-acc4-15b05e869923\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T09:02:34.406522828Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T09:02:43.014191999Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0008428a0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Pragma\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADAEAST:20260402T090245Z:f15f4ae6-036f-4104-9d3c-5b007bb48b61\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Length\": [\"499\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadaeast/b395eb27-1a6a-4aac-b717-f00f205f4a7c\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n \"X-Ms-Correlation-Request-Id\": [\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9d240f3596a0d7016c609b7924c0509e3c0ca3a68bbc6af54e7ce19b76a8b991","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3c4e96966448d637c208e44979ef0fa51eeeabc8718ba70e97e69f046d5f204f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","occurred_at":"2026-04-02T08:48:43Z","failed_at":"e2e","signature_id":"e3fcff74c9339554323e999ccbc70bc28db346f3dda20bfd1db933da9e84e721","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b42240\u003e: \n failed to create HCP cluster cluster-zstream-4-19-vq7bzx: failed waiting for cluster=\"cluster-zstream-4-19-vq7bzx\" in resourcegroup=\"rg-zstream-upgrade-4-19-rjrsjm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d9376b46-84bf-4503-b92d-26a4202504a1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d9376b46-84bf-4503-b92d-26a4202504a1\",\n \"name\": \"d9376b46-84bf-4503-b92d-26a4202504a1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T09:03:10.501037893Z\",\n \"endTime\": \"2026-04-02T09:03:33.384537422Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-vq7bzx: failed waiting for cluster=\\\"cluster-zstream-4-19-vq7bzx\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-rjrsjm\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d9376b46-84bf-4503-b92d-26a4202504a1\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d9376b46-84bf-4503-b92d-26a4202504a1\\\",\\n \\\"name\\\": \\\"d9376b46-84bf-4503-b92d-26a4202504a1\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T09:03:10.501037893Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T09:03:33.384537422Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000b42220\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-19-vq7bzx\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-rjrsjm\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d9376b46-84bf-4503-b92d-26a4202504a1\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d9376b46-84bf-4503-b92d-26a4202504a1\\\",\\n \\\"name\\\": \\\"d9376b46-84bf-4503-b92d-26a4202504a1\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T09:03:10.501037893Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T09:03:33.384537422Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000fba6f0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Msedge-Ref\": [\n \"Ref A: 76C9DB92CECA4A7F9DE40C0EAE9E1D56 Ref B: CHI261140512052 Ref C: 2026-04-02T09:03:41Z\",\n ],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"1422b14c-34d2-442e-88de-df672336bbff\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Envoy-Upstream-Service-Time\": [\"18\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/b1a63070-9329-4e22-b9b2-867d3a3e07c9\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n \"X-Ms-Correlation-Re...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000b42240\u003e: \n failed to create HCP cluster cluster-zstream-4-19-vq7bzx: failed waiting for cluster=\"cluster-zstream-4-19-vq7bzx\" in resourcegroup=\"rg-zstream-upgrade-4-19-rjrsjm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d9376b46-84bf-4503-b92d-26a4202504a1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d9376b46-84bf-4503-b92d-26a4202504a1\",\n \"name\": \"d9376b46-84bf-4503-b92d-26a4202504a1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T09:03:10.501037893Z\",\n \"endTime\": \"2026-04-02T09:03:33.384537422Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-19-vq7bzx: failed waiting for cluster=\\\"cluster-zstream-4-19-vq7bzx\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-rjrsjm\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d9376b46-84bf-4503-b92d-26a4202504a1\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d9376b46-84bf-4503-b92d-26a4202504a1\\\",\\n \\\"name\\\": \\\"d9376b46-84bf-4503-b92d-26a4202504a1\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T09:03:10.501037893Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T09:03:33.384537422Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000b42220\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-19-vq7bzx\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-rjrsjm\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d9376b46-84bf-4503-b92d-26a4202504a1\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d9376b46-84bf-4503-b92d-26a4202504a1\\\",\\n \\\"name\\\": \\\"d9376b46-84bf-4503-b92d-26a4202504a1\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T09:03:10.501037893Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T09:03:33.384537422Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000fba6f0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Msedge-Ref\": [\n \"Ref A: 76C9DB92CECA4A7F9DE40C0EAE9E1D56 Ref B: CHI261140512052 Ref C: 2026-04-02T09:03:41Z\",\n ],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"1422b14c-34d2-442e-88de-df672336bbff\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Envoy-Upstream-Service-Time\": [\"18\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/b1a63070-9329-4e22-b9b2-867d3a3e07c9\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n \"X-Ms-Correlation-Re...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9d240f3596a0d7016c609b7924c0509e3c0ca3a68bbc6af54e7ce19b76a8b991","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"47259aba21bc895d463b92cea1d25157f48fdcd12aaeaf5bd39b8c3a2e7150cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","occurred_at":"2026-04-02T08:48:43Z","failed_at":"e2e","signature_id":"3a2e25e9d1890483fd272049272851235e7e4ec464629137b0a1a3d7b8980e7f","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00075c1e0\u003e: \n failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-wvg7xj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-wvg7xj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: ServiceUnavailable\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ServiceUnavailable\",\n \"message\": \"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster patch-name-cluster: failed starting cluster creation \\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-wvg7xj\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-wvg7xj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"ServiceUnavailable\\\",\\n \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00075c1c0\u003e{\n msg: \"failed starting cluster creation \\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-wvg7xj\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-wvg7xj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"ServiceUnavailable\\\",\\n \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00004d2f0\u003e{\n ErrorCode: \"ServiceUnavailable\",\n StatusCode: 503,\n RawResponse: {\n Status: \"503 Service Unavailable\",\n StatusCode: 503,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Envoy-Upstream-Service-Time\": [\"2878\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"0b9741c0-9777-4781-b4fb-bf1afce41505\",\n ],\n \"Retry-After\": [\"59\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 08:57:47 GMT\",\n ],\n \"Content-Length\": [\"257\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"ServiceUnavailable\"],\n \"X-Ms-Request-Id\": [\n \"87531355-f28f-4c12-bf8b-c7f8a31db6ce\",\n ],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/7ad18b62-8874-4d58-81d6-01921cd42fc7\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260402T085747Z:0b9741c0-9777-4781-b4fb-bf1afce41505\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00075c1e0\u003e: \n failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-wvg7xj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-wvg7xj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: ServiceUnavailable\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ServiceUnavailable\",\n \"message\": \"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster patch-name-cluster: failed starting cluster creation \\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-wvg7xj\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-wvg7xj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"ServiceUnavailable\\\",\\n \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00075c1c0\u003e{\n msg: \"failed starting cluster creation \\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-wvg7xj\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-wvg7xj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"ServiceUnavailable\\\",\\n \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00004d2f0\u003e{\n ErrorCode: \"ServiceUnavailable\",\n StatusCode: 503,\n RawResponse: {\n Status: \"503 Service Unavailable\",\n StatusCode: 503,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Envoy-Upstream-Service-Time\": [\"2878\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"0b9741c0-9777-4781-b4fb-bf1afce41505\",\n ],\n \"Retry-After\": [\"59\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 08:57:47 GMT\",\n ],\n \"Content-Length\": [\"257\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"ServiceUnavailable\"],\n \"X-Ms-Request-Id\": [\n \"87531355-f28f-4c12-bf8b-c7f8a31db6ce\",\n ],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/7ad18b62-8874-4d58-81d6-01921cd42fc7\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260402T085747Z:0b9741c0-9777-4781-b4fb-bf1afce41505\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"47c4fe17c8b40b438461015820e355e731ca2f90428cea3ac600911bb9a03df7","failure_pattern":"ERROR CODE: ServiceUnavailable; detail message Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ServiceUnavailable"}},{"row_id":"59c030e4f0f18a0efcc3bb1e50bec1ec451b488543085e87dcc10ec086187ec0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","occurred_at":"2026-04-02T08:48:43Z","failed_at":"e2e","signature_id":"cacb304bc7bd5ba62586475b5881017dd5ae0a6ac7b13d88e7aa4f249a8ed2ea","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:123]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0004e6700\u003e: \n failed starting cluster creation \"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-p6dr85\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-p6dr85/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster2\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: ServiceUnavailable\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ServiceUnavailable\",\n \"message\": \"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed starting cluster creation \\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-p6dr85\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-p6dr85/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster2\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"ServiceUnavailable\\\",\\n \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001402630\u003e{\n ErrorCode: \"ServiceUnavailable\",\n StatusCode: 503,\n RawResponse: {\n Status: \"503 Service Unavailable\",\n StatusCode: 503,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"981e3b54-276e-45f0-a48f-b05f015c79dc\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"8461\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/d750dcff-9cd4-44ec-8a65-6f1b69e57a9d\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"45bdda71-4b5b-4b9b-9fd7-4df364bf7808\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260402T085811Z:45bdda71-4b5b-4b9b-9fd7-4df364bf7808\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"257\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\"ServiceUnavailable\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: C36632182EBE4B8EB52F9106BF8C5BB2 Ref B: CHI261140502029 Ref C: 2026-04-02T08:58:02Z\",\n ],\n \"Retry-After\": [\"59\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 08:58:11 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0004e6480\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 83, 101, 114, 118, 105, 99, 101, 85, 110, 97, 118, 97, 105, 108, 97, 98, 108, 101, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 108, 117, 115, 116, 101, 114, 32, 105, 110, 115, 116, 97, 108, 108, 97, 116, 105, 111, 110, 32, 104, 97, 115, 32, 102, 97, 105, 108, 101, 100, 32, 100, 117, 101, 32, 116, 111, 32, 117, 110, 97, 118, 97, 105, 108, 97, 98, 108, 101, 32, 99, 97, 112, 97, 99, 105, 116, 121, 46, 32, 87, 104, 105, 108, 101, 32, 119, 101, 32, 97, 114, 101, 32, 98, 114, 105, 110, 103, 105, 110, 103, 32, 111, 110, 32, 97, 100, 100, 105, 116, 105, 111, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:123]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0004e6700\u003e: \n failed starting cluster creation \"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-p6dr85\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-p6dr85/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster2\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: ServiceUnavailable\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ServiceUnavailable\",\n \"message\": \"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed starting cluster creation \\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-p6dr85\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-p6dr85/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster2\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"ServiceUnavailable\\\",\\n \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001402630\u003e{\n ErrorCode: \"ServiceUnavailable\",\n StatusCode: 503,\n RawResponse: {\n Status: \"503 Service Unavailable\",\n StatusCode: 503,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"981e3b54-276e-45f0-a48f-b05f015c79dc\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"8461\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/d750dcff-9cd4-44ec-8a65-6f1b69e57a9d\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"45bdda71-4b5b-4b9b-9fd7-4df364bf7808\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260402T085811Z:45bdda71-4b5b-4b9b-9fd7-4df364bf7808\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"257\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\"ServiceUnavailable\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: C36632182EBE4B8EB52F9106BF8C5BB2 Ref B: CHI261140502029 Ref C: 2026-04-02T08:58:02Z\",\n ],\n \"Retry-After\": [\"59\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 08:58:11 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0004e6480\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 83, 101, 114, 118, 105, 99, 101, 85, 110, 97, 118, 97, 105, 108, 97, 98, 108, 101, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 108, 117, 115, 116, 101, 114, 32, 105, 110, 115, 116, 97, 108, 108, 97, 116, 105, 111, 110, 32, 104, 97, 115, 32, 102, 97, 105, 108, 101, 100, 32, 100, 117, 101, 32, 116, 111, 32, 117, 110, 97, 118, 97, 105, 108, 97, 98, 108, 101, 32, 99, 97, 112, 97, 99, 105, 116, 121, 46, 32, 87, 104, 105, 108, 101, 32, 119, 101, 32, 97, 114, 101, 32, 98, 114, 105, 110, 103, 105, 110, 103, 32, 111, 110, 32, 97, 100, 100, 105, 116, 105, 111, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"47c4fe17c8b40b438461015820e355e731ca2f90428cea3ac600911bb9a03df7","failure_pattern":"ERROR CODE: ServiceUnavailable; detail message Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ServiceUnavailable"}},{"row_id":"d09a16eb19062aab3c36f295e08f9a5249cf65c72e6a7ecd358fdfd1be86845f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","occurred_at":"2026-04-02T08:48:43Z","failed_at":"e2e","signature_id":"6e970d09796e67482e58fb3616052cfbda9bac0d41b02e47fa38c4737f014b34","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0007b6ba0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-59tjhl: failed waiting for cluster=\"cluster-zstream-4-21-59tjhl\" in resourcegroup=\"rg-zstream-upgrade-4-21-rddnsg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bab93702-2495-45ca-bdfe-fef91e9947ec\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bab93702-2495-45ca-bdfe-fef91e9947ec\",\n \"name\": \"bab93702-2495-45ca-bdfe-fef91e9947ec\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T09:02:38.689735931Z\",\n \"endTime\": \"2026-04-02T09:02:55.541508676Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-21-59tjhl: failed waiting for cluster=\\\"cluster-zstream-4-21-59tjhl\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-rddnsg\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bab93702-2495-45ca-bdfe-fef91e9947ec\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bab93702-2495-45ca-bdfe-fef91e9947ec\\\",\\n \\\"name\\\": \\\"bab93702-2495-45ca-bdfe-fef91e9947ec\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T09:02:38.689735931Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T09:02:55.541508676Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0007b6b00\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-59tjhl\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-rddnsg\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bab93702-2495-45ca-bdfe-fef91e9947ec\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bab93702-2495-45ca-bdfe-fef91e9947ec\\\",\\n \\\"name\\\": \\\"bab93702-2495-45ca-bdfe-fef91e9947ec\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T09:02:38.689735931Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T09:02:55.541508676Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000f46930\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"499\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Cache-Control\": [\"no-cache\"],\n \"X-Ms-Request-Id\": [\n \"c629973c-9d73-46a2-a7e1-a7d37a1a5a9a\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"41\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/e829b8ea-8772-4c59-8a5b-1fa05e5027c2\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n \"X-Msedge-Ref\": [\n \"Ref A: F83DD0A50C8A422096F238C28BB336A7 Ref B: CHI261140513031 Ref C: 2026-04-02T09:02:59Z\",\n ],\n \"Expire...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0007b6ba0\u003e: \n failed to create HCP cluster cluster-zstream-4-21-59tjhl: failed waiting for cluster=\"cluster-zstream-4-21-59tjhl\" in resourcegroup=\"rg-zstream-upgrade-4-21-rddnsg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bab93702-2495-45ca-bdfe-fef91e9947ec\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bab93702-2495-45ca-bdfe-fef91e9947ec\",\n \"name\": \"bab93702-2495-45ca-bdfe-fef91e9947ec\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T09:02:38.689735931Z\",\n \"endTime\": \"2026-04-02T09:02:55.541508676Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-21-59tjhl: failed waiting for cluster=\\\"cluster-zstream-4-21-59tjhl\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-rddnsg\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bab93702-2495-45ca-bdfe-fef91e9947ec\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bab93702-2495-45ca-bdfe-fef91e9947ec\\\",\\n \\\"name\\\": \\\"bab93702-2495-45ca-bdfe-fef91e9947ec\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T09:02:38.689735931Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T09:02:55.541508676Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0007b6b00\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-59tjhl\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-rddnsg\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bab93702-2495-45ca-bdfe-fef91e9947ec\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bab93702-2495-45ca-bdfe-fef91e9947ec\\\",\\n \\\"name\\\": \\\"bab93702-2495-45ca-bdfe-fef91e9947ec\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T09:02:38.689735931Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T09:02:55.541508676Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000f46930\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"499\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Cache-Control\": [\"no-cache\"],\n \"X-Ms-Request-Id\": [\n \"c629973c-9d73-46a2-a7e1-a7d37a1a5a9a\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"41\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/e829b8ea-8772-4c59-8a5b-1fa05e5027c2\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n \"X-Msedge-Ref\": [\n \"Ref A: F83DD0A50C8A422096F238C28BB336A7 Ref B: CHI261140513031 Ref C: 2026-04-02T09:02:59Z\",\n ],\n \"Expire...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9d240f3596a0d7016c609b7924c0509e3c0ca3a68bbc6af54e7ce19b76a8b991","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["47c4fe17c8b40b438461015820e355e731ca2f90428cea3ac600911bb9a03df7","9d240f3596a0d7016c609b7924c0509e3c0ca3a68bbc6af54e7ce19b76a8b991"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039575514619842560","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-02T05:27:45Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-02T01:37:08Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"0e6b2cac3a3ece507181a5e0266572ab78ddc000dbd5ae73fe42eb4a9e5817a0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","failed_at":"e2e","signature_id":"7a56671f4aff56ecc03b701290aceff3662c37535980cb96bf3f4b76002fa5d3","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:118]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001418640\u003e: \n failed creating external auth \"ea-list\" in resourcegroup=\"ea-list-sqv9wn\" for cluster=\"ea-list4lvwkr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-sqv9wn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-list4lvwkr/externalAuths/ea-list\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed creating external auth \\\"ea-list\\\" in resourcegroup=\\\"ea-list-sqv9wn\\\" for cluster=\\\"ea-list4lvwkr\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-sqv9wn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-list4lvwkr/externalAuths/ea-list\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001366990\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Expires\": [\"-1\"],\n \"X-Ms-Request-Id\": [\n \"f4e61ff2-b02c-4f78-930a-516f5367173c\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 02:10:05 GMT\",\n ],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 04AE8E5AF86B478685A5B528872AB82B Ref B: CHI261140505062 Ref C: 2026-04-02T02:09:59Z\",\n ],\n \"Content-Length\": [\"107\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/786cdfbd-1a09-42c1-be7d-896a3083330d\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"4be39819-497a-401e-a7b1-442d6a3bb89a\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"7051\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260402T021006Z:4be39819-497a-401e-a7b1-442d6a3bb89a\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc001418500\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:118]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001418640\u003e: \n failed creating external auth \"ea-list\" in resourcegroup=\"ea-list-sqv9wn\" for cluster=\"ea-list4lvwkr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-sqv9wn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-list4lvwkr/externalAuths/ea-list\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed creating external auth \\\"ea-list\\\" in resourcegroup=\\\"ea-list-sqv9wn\\\" for cluster=\\\"ea-list4lvwkr\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-sqv9wn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-list4lvwkr/externalAuths/ea-list\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001366990\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Expires\": [\"-1\"],\n \"X-Ms-Request-Id\": [\n \"f4e61ff2-b02c-4f78-930a-516f5367173c\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 02:10:05 GMT\",\n ],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 04AE8E5AF86B478685A5B528872AB82B Ref B: CHI261140505062 Ref C: 2026-04-02T02:09:59Z\",\n ],\n \"Content-Length\": [\"107\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/786cdfbd-1a09-42c1-be7d-896a3083330d\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"4be39819-497a-401e-a7b1-442d6a3bb89a\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"7051\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260402T021006Z:4be39819-497a-401e-a7b1-442d6a3bb89a\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc001418500\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2e5dafa9063ca1b7d5e22d21e90655c2cdfe7f0346c9bc2f4207e91f569aa8e6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2047ccce6ff06d7921325b325902b52c9bd4e30b3ecb67721376aa033bd37e25","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","failed_at":"e2e","signature_id":"ae1b83127fad3f1bb11977b594bb276b77277cef01ce027657dee63ae31ce624","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0017d0f90\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-lvdt77\" in resourcegroup=\"admin-credential-lifecycle-test-cdl8dq\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-lvdt77 in resource group admin-credential-lifecycle-test-cdl8dq, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-lvdt77\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-cdl8dq\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-lvdt77 in resource group admin-credential-lifecycle-test-cdl8dq, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0000d0170\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-lvdt77 in resource group admin-credential-lifecycle-test-cdl8dq\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0017d0f90\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-lvdt77\" in resourcegroup=\"admin-credential-lifecycle-test-cdl8dq\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-lvdt77 in resource group admin-credential-lifecycle-test-cdl8dq, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-lvdt77\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-cdl8dq\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-lvdt77 in resource group admin-credential-lifecycle-test-cdl8dq, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0000d0170\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-lvdt77 in resource group admin-credential-lifecycle-test-cdl8dq\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}},{"row_id":"225e8879c57e9e0aa45798571b0022b66f3199828ce3e40aba09a4e55405a589","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","failed_at":"e2e","signature_id":"ad02bdcbba8a2cd55fe28294470d643c39c011c3cd16fba715a90713c1dd2990","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:103]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00142ee20\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-4qqv6m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: LockContention\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LockContention\",\n \"message\": \"Failed to acquire lock: timed out\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-4qqv6m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"LockContention\\\",\\n \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00104d530\u003e{\n ErrorCode: \"LockContention\",\n StatusCode: 503,\n RawResponse: {\n Status: \"503 Service Unavailable\",\n StatusCode: 503,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/0949565f-2116-4c2d-ac42-f874e16826f1\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260402T021009Z:c60ff00c-0498-471e-83f9-643c3970b3df\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 84A2B7061A1D427CAC14B9844929E76E Ref B: CHI261140502054 Ref C: 2026-04-02T02:09:58Z\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"186\"],\n \"Retry-After\": [\"10\"],\n \"X-Ms-Error-Code\": [\"LockContention\"],\n \"X-Ms-Request-Id\": [\n \"342b364f-60a1-493c-a980-5f75e6f1c77f\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"10010\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 02:10:08 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"c60ff00c-0498-471e-83f9-643c3970b3df\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00142ed60\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 76, 111, 99, 107, 67, 111, 110, 116, 101, 110, 116, 105, 111, 110, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 32, 116, 111, 32, 97, 99, 113, 117, 105, 114, 101, 32, 108, 111, 99, 107, 58, 32, 116, 105, 109, 101, 100, 32, 111, 117, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 57, 51, 57, 57, 50, 56, 49, 45, 48, 48, 97, 50, 45, 52, 98, 51, 57, 45, 98, 98, 51, 100, 45, 98, 50, 54, 52, 53, 98, 98, 98, 100, 98, 57, 51, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n Co...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:103]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00142ee20\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-4qqv6m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: LockContention\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LockContention\",\n \"message\": \"Failed to acquire lock: timed out\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-4qqv6m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"LockContention\\\",\\n \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00104d530\u003e{\n ErrorCode: \"LockContention\",\n StatusCode: 503,\n RawResponse: {\n Status: \"503 Service Unavailable\",\n StatusCode: 503,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/0949565f-2116-4c2d-ac42-f874e16826f1\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260402T021009Z:c60ff00c-0498-471e-83f9-643c3970b3df\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 84A2B7061A1D427CAC14B9844929E76E Ref B: CHI261140502054 Ref C: 2026-04-02T02:09:58Z\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"186\"],\n \"Retry-After\": [\"10\"],\n \"X-Ms-Error-Code\": [\"LockContention\"],\n \"X-Ms-Request-Id\": [\n \"342b364f-60a1-493c-a980-5f75e6f1c77f\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"10010\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 02:10:08 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"c60ff00c-0498-471e-83f9-643c3970b3df\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00142ed60\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 76, 111, 99, 107, 67, 111, 110, 116, 101, 110, 116, 105, 111, 110, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 32, 116, 111, 32, 97, 99, 113, 117, 105, 114, 101, 32, 108, 111, 99, 107, 58, 32, 116, 105, 109, 101, 100, 32, 111, 117, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 57, 51, 57, 57, 50, 56, 49, 45, 48, 48, 97, 50, 45, 52, 98, 51, 57, 45, 98, 98, 51, 100, 45, 98, 50, 54, 52, 53, 98, 98, 98, 100, 98, 57, 51, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n Co...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e17b72ff49490f6e1940fa5deb5057f7012ebd4b2cee2f2d2e8b8d47e6784bb9","failure_pattern":"ERROR CODE: LockContention; detail message Failed to acquire lock: timed out; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LockContention"}},{"row_id":"3d487e517d63524b436b59f851e60969693ce3d17244e0f9e9f95d94551304f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","failed_at":"e2e","signature_id":"efed37f596e7501dc50ed6fc82727bd5d2db1dd41b58a065c7155228aa4d0495","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:159]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a814e0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-n7b9pd-pbv84h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-n7b9pd/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-n7b9pd-pbv84h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-n7b9pd/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000bd97a0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260402T022003Z:0a43673e-6459-4a7a-9684-1ffc756dffcb\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 4F8CAF08C1B3477AAEC3FF6E1D6910E3 Ref B: CHI261140505040 Ref C: 2026-04-02T02:20:02Z\",\n ],\n \"Date\": [\n \"Thu, 02 Apr 2026 02:20:02 GMT\",\n ],\n \"Expires\": [\"-1\"],\n \"X-Envoy-Upstream-Service-Time\": [\"663\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/91a17ebb-245e-4f3f-89d3-25fb7c6bd722\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Content-Length\": [\"107\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"66ee99e3-1f96-460f-b64b-d03b134f3ec4\",\n ],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"Cache-Control\": [\"no-cache\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"0a43673e-6459-4a7a-9684-1ffc756dffcb\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a81420\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:159]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a814e0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-n7b9pd-pbv84h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-n7b9pd/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-n7b9pd-pbv84h/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-n7b9pd/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000bd97a0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260402T022003Z:0a43673e-6459-4a7a-9684-1ffc756dffcb\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 4F8CAF08C1B3477AAEC3FF6E1D6910E3 Ref B: CHI261140505040 Ref C: 2026-04-02T02:20:02Z\",\n ],\n \"Date\": [\n \"Thu, 02 Apr 2026 02:20:02 GMT\",\n ],\n \"Expires\": [\"-1\"],\n \"X-Envoy-Upstream-Service-Time\": [\"663\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/91a17ebb-245e-4f3f-89d3-25fb7c6bd722\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Content-Length\": [\"107\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"66ee99e3-1f96-460f-b64b-d03b134f3ec4\",\n ],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"Cache-Control\": [\"no-cache\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"0a43673e-6459-4a7a-9684-1ffc756dffcb\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a81420\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2e5dafa9063ca1b7d5e22d21e90655c2cdfe7f0346c9bc2f4207e91f569aa8e6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"464aaf9fcf051ebbab8bac04f746ff178e536ba8d4270e9917463881df6f4a59","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","failed_at":"e2e","signature_id":"6eacec424ad1161d7aacbab03f8a93ea1c58263a875329c1260729344930c215","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:147]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008c23a0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-c4ph7r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-pw4hx4/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-c4ph7r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-pw4hx4/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00031b110\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260402T022005Z:bae2f98c-c749-481d-9054-9921e081945e\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/62d919b4-4225-4967-9df6-08b0b5351023\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 02:20:05 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"4c5f9568-bb9b-40c3-81ad-0ca630f99d24\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: 2552FC52616C4D5BB46364A7B7C0692E Ref B: CH1AA2020617017 Ref C: 2026-04-02T02:20:02Z\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"2628\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"Content-Length\": [\"107\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"bae2f98c-c749-481d-9054-9921e081945e\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0008c21e0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:147]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008c23a0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-c4ph7r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-pw4hx4/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-c4ph7r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-pw4hx4/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00031b110\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260402T022005Z:bae2f98c-c749-481d-9054-9921e081945e\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/62d919b4-4225-4967-9df6-08b0b5351023\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 02:20:05 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"4c5f9568-bb9b-40c3-81ad-0ca630f99d24\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: 2552FC52616C4D5BB46364A7B7C0692E Ref B: CH1AA2020617017 Ref C: 2026-04-02T02:20:02Z\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"2628\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"Content-Length\": [\"107\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"bae2f98c-c749-481d-9054-9921e081945e\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0008c21e0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2e5dafa9063ca1b7d5e22d21e90655c2cdfe7f0346c9bc2f4207e91f569aa8e6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"66cbdd7a97979ed356b1bb1fd18e8d02ec269dd426bef7fa4262764bf88f3630","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","failed_at":"e2e","signature_id":"90bf4c501cbcfdc4b92b5c6d62b79255bb6cb4ec6dd2de919de62549dec49677","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f5e7b0\u003e: \n failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-qcjg4q\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-qcjg4q, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-qcjg4q\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-qcjg4q, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0015ab170\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-qcjg4q\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f5e7b0\u003e: \n failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-qcjg4q\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-qcjg4q, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-qcjg4q\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-qcjg4q, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0015ab170\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-qcjg4q\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}},{"row_id":"7d0b2dbe92159cb96573e11ebb01e518ffce36af353f5b65e1317fc3b03e5521","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","failed_at":"e2e","signature_id":"d3286a20e487ac27f27dee83ac29804121c9b49a8c8b8ea4d003f723f9be0d69","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:159]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00080bb60\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-h6h9kt-6zms4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-h6h9kt/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-h6h9kt-6zms4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-h6h9kt/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000792300\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Content-Length\": [\"107\"],\n \"Expires\": [\"-1\"],\n \"X-Envoy-Upstream-Service-Time\": [\"339\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/db8b7c42-a699-4749-a2d0-f6d8f859aacd\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"35b54f7f-739e-4203-895d-a04b57b209f1\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Ms-Request-Id\": [\n \"dd3c2403-c919-40cc-afa1-b2bd95e920f7\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 8A45806B62A847B0A86E8C8C1276610B Ref B: CHI261140508031 Ref C: 2026-04-02T02:24:34Z\",\n ],\n \"Date\": [\n \"Thu, 02 Apr 2026 02:24:34 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260402T022434Z:35b54f7f-739e-4203-895d-a04b57b209f1\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00080b860\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:159]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00080bb60\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-h6h9kt-6zms4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-h6h9kt/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-h6h9kt-6zms4z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-h6h9kt/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000792300\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Content-Length\": [\"107\"],\n \"Expires\": [\"-1\"],\n \"X-Envoy-Upstream-Service-Time\": [\"339\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/db8b7c42-a699-4749-a2d0-f6d8f859aacd\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"35b54f7f-739e-4203-895d-a04b57b209f1\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Ms-Request-Id\": [\n \"dd3c2403-c919-40cc-afa1-b2bd95e920f7\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 8A45806B62A847B0A86E8C8C1276610B Ref B: CHI261140508031 Ref C: 2026-04-02T02:24:34Z\",\n ],\n \"Date\": [\n \"Thu, 02 Apr 2026 02:24:34 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260402T022434Z:35b54f7f-739e-4203-895d-a04b57b209f1\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00080b860\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2e5dafa9063ca1b7d5e22d21e90655c2cdfe7f0346c9bc2f4207e91f569aa8e6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8d3e30a14d6c28053120cb144d51e871ba766d70785ade3f9c5035732340b12e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","failed_at":"e2e","signature_id":"61fec9e174cbe2103641b0ca2d00e068a13dee8925feb67a852f2d37175153ed","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0009f2840\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-k2qhj7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: LockContention\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LockContention\",\n \"message\": \"Failed to acquire lock: timed out\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-k2qhj7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"LockContention\\\",\\n \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00159e990\u003e{\n ErrorCode: \"LockContention\",\n StatusCode: 503,\n RawResponse: {\n Status: \"503 Service Unavailable\",\n StatusCode: 503,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/1c5d69eb-b218-43c3-89e1-797546221c53\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 02:10:08 GMT\",\n ],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"LockContention\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260402T021008Z:afdf4447-88c0-4010-85eb-2250eadd30fe\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: 1EC287F8926644E099B85CCB3B5E9590 Ref B: CHI261140504042 Ref C: 2026-04-02T02:09:58Z\",\n ],\n \"Content-Length\": [\"186\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Request-Id\": [\n \"2b953333-0bff-468f-ad05-d66baee5ebf5\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"10006\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"afdf4447-88c0-4010-85eb-2250eadd30fe\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"Retry-After\": [\"10\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0009f2700\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 76, 111, 99, 107, 67, 111, 110, 116, 101, 110, 116, 105, 111, 110, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 32, 116, 111, 32, 97, 99, 113, 117, 105, 114, 101, 32, 108, 111, 99, 107, 58, 32, 116, 105, 109, 101, 100, 32, 111, 117, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 57, 51, 57, 57, 50, 56, 49, 45, 48, 48, 97, 50, 45, 52, 98, 51, 57, 45, 98, 98, 51, 100, 45, 98, 50, 54, 52, 53, 98, 98, 98, 100, 98, 57, 51, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n Conte...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0009f2840\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-k2qhj7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: LockContention\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LockContention\",\n \"message\": \"Failed to acquire lock: timed out\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-k2qhj7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"LockContention\\\",\\n \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00159e990\u003e{\n ErrorCode: \"LockContention\",\n StatusCode: 503,\n RawResponse: {\n Status: \"503 Service Unavailable\",\n StatusCode: 503,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/1c5d69eb-b218-43c3-89e1-797546221c53\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 02:10:08 GMT\",\n ],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\"LockContention\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260402T021008Z:afdf4447-88c0-4010-85eb-2250eadd30fe\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: 1EC287F8926644E099B85CCB3B5E9590 Ref B: CHI261140504042 Ref C: 2026-04-02T02:09:58Z\",\n ],\n \"Content-Length\": [\"186\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Request-Id\": [\n \"2b953333-0bff-468f-ad05-d66baee5ebf5\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"10006\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"afdf4447-88c0-4010-85eb-2250eadd30fe\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"Retry-After\": [\"10\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0009f2700\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 76, 111, 99, 107, 67, 111, 110, 116, 101, 110, 116, 105, 111, 110, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 32, 116, 111, 32, 97, 99, 113, 117, 105, 114, 101, 32, 108, 111, 99, 107, 58, 32, 116, 105, 109, 101, 100, 32, 111, 117, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 57, 51, 57, 57, 50, 56, 49, 45, 48, 48, 97, 50, 45, 52, 98, 51, 57, 45, 98, 98, 51, 100, 45, 98, 50, 54, 52, 53, 98, 98, 98, 100, 98, 57, 51, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n Conte...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e17b72ff49490f6e1940fa5deb5057f7012ebd4b2cee2f2d2e8b8d47e6784bb9","failure_pattern":"ERROR CODE: LockContention; detail message Failed to acquire lock: timed out; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LockContention"}},{"row_id":"b462a16c4db5e876c561fddc50406e10102fbc3a414af7caa866bad1664a897d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","failed_at":"e2e","signature_id":"afe25b897c006e3672b59622d300b23541d7af3aa6c8e6237eeb004b4e3541ee","test_name":"Customer should be able to create an HCP cluster","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:106]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011440f0\u003e: \n failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-zzfq7g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-zzfq7g, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-zzfq7g\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-zzfq7g, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ef19b0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-zzfq7g\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:106]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011440f0\u003e: \n failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-zzfq7g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-zzfq7g, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-zzfq7g\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-zzfq7g, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ef19b0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-zzfq7g\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}},{"row_id":"c9642800496df94b1df166a5c4b120e21cc6516b02e7cdd97d9d9c14a959be02","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","failed_at":"e2e","signature_id":"8a5c03c98dbcaa4cc970195ad454045945fe07e39d81c73216bb358fdf6301fc","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009b8fc0\u003e: \n failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-7ntzdq\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-7ntzdq, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-7ntzdq\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-7ntzdq, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000676740\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-7ntzdq\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009b8fc0\u003e: \n failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-7ntzdq\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-7ntzdq, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-7ntzdq\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-7ntzdq, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000676740\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-7ntzdq\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded"}},{"row_id":"efcfd26fc6920c54fa1362b1b84822a479d0e202510434ba5308fc387771cd0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","failed_at":"e2e","signature_id":"4d26299f52b0b8986f5cb45926b1e2da9613d8a375ee0a382f06b7e2c7a01945","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:181]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0012b0780\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-8kqbht/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-8kqbht/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012f34a0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Pragma\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Content-Length\": [\"107\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/1f894e75-bf9b-4f38-b36f-086c2d1ebd50\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: 49D6E7E0AC224A1688162351432A0445 Ref B: CH1AA2020610019 Ref C: 2026-04-02T02:19:06Z\",\n ],\n \"Date\": [\n \"Thu, 02 Apr 2026 02:19:06 GMT\",\n ],\n \"X-Ms-Request-Id\": [\n \"4c158d49-d82a-452b-867b-1015459e1703\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260402T021907Z:0ff2a06f-4992-4cf9-ae72-86b939159372\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Cache-Control\": [\"no-cache\"],\n \"X-Envoy-Upstream-Service-Time\": [\"399\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"0ff2a06f-4992-4cf9-ae72-86b939159372\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0012b06c0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"manageme...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:181]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0012b0780\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-8kqbht/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-8kqbht/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012f34a0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Pragma\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Content-Length\": [\"107\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/1f894e75-bf9b-4f38-b36f-086c2d1ebd50\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: 49D6E7E0AC224A1688162351432A0445 Ref B: CH1AA2020610019 Ref C: 2026-04-02T02:19:06Z\",\n ],\n \"Date\": [\n \"Thu, 02 Apr 2026 02:19:06 GMT\",\n ],\n \"X-Ms-Request-Id\": [\n \"4c158d49-d82a-452b-867b-1015459e1703\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260402T021907Z:0ff2a06f-4992-4cf9-ae72-86b939159372\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Cache-Control\": [\"no-cache\"],\n \"X-Envoy-Upstream-Service-Time\": [\"399\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"0ff2a06f-4992-4cf9-ae72-86b939159372\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0012b06c0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"manageme...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2e5dafa9063ca1b7d5e22d21e90655c2cdfe7f0346c9bc2f4207e91f569aa8e6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fb2481e1c34e5db8a0ea61310e7d5b117570bab8c846f7cec5be1d76f0d54800","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","failed_at":"e2e","signature_id":"a065c8647285da4ae861666481d317b07db2efdc79edc92e410f5b8bacb3f1ae","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:192]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0011d2100\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-t8jz2f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-t8jz2f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00086f3e0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/58e1c38c-4da4-44dd-994d-8cbc32ac06f7\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"3536f45d-afc7-473f-8b37-bad9de7684d7\",\n ],\n \"Date\": [\n \"Thu, 02 Apr 2026 02:22:32 GMT\",\n ],\n \"Content-Type\": [\"application/json\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Msedge-Ref\": [\n \"Ref A: F4CED9AE58F244F885602E16952E2D6D Ref B: CH1AA2020617037 Ref C: 2026-04-02T02:22:31Z\",\n ],\n \"X-Ms-Request-Id\": [\n \"f2afc4a3-ed79-4022-adac-dc064151bd9a\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"744\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260402T022232Z:3536f45d-afc7-473f-8b37-bad9de7684d7\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"107\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0011d2040\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"mana...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:192]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0011d2100\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-t8jz2f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-t8jz2f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kv-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00086f3e0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/58e1c38c-4da4-44dd-994d-8cbc32ac06f7\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"3536f45d-afc7-473f-8b37-bad9de7684d7\",\n ],\n \"Date\": [\n \"Thu, 02 Apr 2026 02:22:32 GMT\",\n ],\n \"Content-Type\": [\"application/json\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Msedge-Ref\": [\n \"Ref A: F4CED9AE58F244F885602E16952E2D6D Ref B: CH1AA2020617037 Ref C: 2026-04-02T02:22:31Z\",\n ],\n \"X-Ms-Request-Id\": [\n \"f2afc4a3-ed79-4022-adac-dc064151bd9a\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"744\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260402T022232Z:3536f45d-afc7-473f-8b37-bad9de7684d7\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"107\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0011d2040\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"mana...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2e5dafa9063ca1b7d5e22d21e90655c2cdfe7f0346c9bc2f4207e91f569aa8e6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["2e5dafa9063ca1b7d5e22d21e90655c2cdfe7f0346c9bc2f4207e91f569aa8e6","61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","e17b72ff49490f6e1940fa5deb5057f7012ebd4b2cee2f2d2e8b8d47e6784bb9"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}