{"meta":{"start_date":"2026-04-02","end_date":"2026-04-02","start_at":"2026-04-02T00:00:00Z","end_at":"2026-04-03T00:00:00Z","timezone":"UTC","generated_at":"2026-08-27T15:27:43Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":6,"failed_runs":5,"raw_occurrences":46,"matched_occurrences":46,"runs_affected":5},"rows":[{"environment":"stg","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-9sv9gw\" in resourcegroup=\"admin-credential-lifecycle-test-g6kp9g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-9sv9gw in resource group admin-credential-lifecycle-test-g6kp9g, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","occurrences":25,"runs_affected":3,"failed_runs":3,"run_impact_percent":50,"anchor_occurrences":25,"after_last_push_count":25,"trend_counts":[0,0,0,0,0,0,25],"trend_range":"2026-03-27..2026-04-02 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-23"],"prior_runs_affected":5,"prior_last_seen_at":"2026-03-23T02:00:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:101]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009decc0\u003e: \n    failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-782d8g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-782d8g, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-np-labels-taints\\\" in resourcegroup=\\\"rg-np-labels-taints-782d8g\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-782d8g, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000c04df0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-782d8g\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:101]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0009decc0\u003e: \n    failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-782d8g\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-782d8g, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-np-labels-taints\\\" in resourcegroup=\\\"rg-np-labels-taints-782d8g\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-782d8g, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000c04df0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-782d8g\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:103]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00156a3c0\u003e: \n    failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-x5nbs9\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-x5nbs9, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-x5nbs9\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-x5nbs9, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000dbddd0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-x5nbs9\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:103]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00156a3c0\u003e: \n    failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-x5nbs9\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-x5nbs9, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-x5nbs9\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-x5nbs9, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000dbddd0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ea-cluster in resource group external-auth-cluster-x5nbs9\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:105]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00071ba70\u003e: \n    failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dmzpzs\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-dmzpzs, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-dmzpzs\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-dmzpzs, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0009c5c60\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-dmzpzs\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:105]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00071ba70\u003e: \n    failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-dmzpzs\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-dmzpzs, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-dmzpzs\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-dmzpzs, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0009c5c60\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-dmzpzs\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039846445451841536","occurred_at":"2026-04-02T23:24:25Z","signature_id":"425cb192426cfae26c27c30de57f81c15b90dc678940b4bca02416c7cb2beaed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039846445451841536","occurred_at":"2026-04-02T23:24:25Z","signature_id":"48700371ca5e3ea6f36170d783d442d058d1cb1424257da18a3fc2d24e97609f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039846445451841536","occurred_at":"2026-04-02T23:24:25Z","signature_id":"70daa82f0e74a2a68d0f694e0626bdd2ee7f59b84ad17694935a98cf2693387c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039846445451841536","occurred_at":"2026-04-02T23:24:25Z","signature_id":"925d5b5e8d1cb7062ad7f97148097b4bbbd90c0a007478dcd253302060efce42","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039846445451841536","occurred_at":"2026-04-02T23:24:25Z","signature_id":"abbe9ccf77f58b9dcecdebc516030b33899c9d00e6c69935ca042cc483f352bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039846445451841536","occurred_at":"2026-04-02T23:24:25Z","signature_id":"daa01e0a071bb23981629322b51a8a838a0a20c385d7cccf1bda9ff5fe01bb42","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039846445451841536","occurred_at":"2026-04-02T23:24:25Z","signature_id":"f62f0cea8a0efc666a80f05794478f17dc1e76b6c132ca2c22eb98b5fbbbd8d1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"1c8fe61fbd43db9169aaa7fbf69609a8728f6d8509220f9a92d6f861ef91f8c0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"3a274d8c90c6db3b46a4ec71e46a64843c4377cb4ead33281af23bd84b9ab97d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"473710e31048fab0665756ba98c3a4020c0308701ea4c9a9babf37c2b2ab4f39","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"4c6fe9b34a98463c74e064da1997e5316b414aefa31a08707da1116847153d72","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"5e59c002f97b94a2b5076e852a21ebe711b44b0942e4ecc5862f23348be56977","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"61f89257fbd679c1f5a3f14ecb2cfdc110c59434b5cb1139ebf64baa58624853","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"6b5479644ee17536d3c82e89ebbb26e7571bb914b168ab9d74f281baf41fd90c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"807f4398a1b973a767b11bdff3ba6ffe1f98c0444aa1aecbf515704d79fa35f2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"87f712f0082578aceb2139869be1b2173936efbf987880da72d2af8ef95f3557","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"96e939a1dc8a1464c8152e2dcfbf00623eaef4723e51466619b88ddbd5291584","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"a77aba6e2f5896ef8dd63e5789b9c1381fb9c0ef40adec0f73550907fb8efca7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"bd8c64743544c200535996997a509a9d5a3ba8e598a513d39001e796b305da6a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"c6f66b20a06674c7be9ba89a2d96b7d1acc8a1ac27f90ece2f2477e63df258d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"f3edf87053a3ddfa94c5d07ba8914a620291e1ca95012ba77d27456596d7a2f7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","signature_id":"8a5c03c98dbcaa4cc970195ad454045945fe07e39d81c73216bb358fdf6301fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","signature_id":"90bf4c501cbcfdc4b92b5c6d62b79255bb6cb4ec6dd2de919de62549dec49677","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","signature_id":"ae1b83127fad3f1bb11977b594bb276b77277cef01ce027657dee63ae31ce624","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","signature_id":"afe25b897c006e3672b59622d300b23541d7af3aa6c8e6237eeb004b4e3541ee","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"9d240f3596a0d7016c609b7924c0509e3c0ca3a68bbc6af54e7ce19b76a8b991","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":10,"runs_affected":2,"failed_runs":2,"run_impact_percent":33.333333333333336,"anchor_occurrences":10,"after_last_push_count":10,"trend_counts":[0,0,0,0,0,0,10],"trend_range":"2026-03-27..2026-04-02 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000732640\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-7mzwzp: failed waiting for cluster=\"cluster-zstream-4-20-7mzwzp\" in resourcegroup=\"rg-zstream-upgrade-4-20-tq69zx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\",\n      \"name\": \"e3ee4618-26eb-473e-867f-a23c3644e2bb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-04-02T17:31:57.472030714Z\",\n      \"endTime\": \"2026-04-02T17:32:55.049606818Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-20-7mzwzp: failed waiting for cluster=\\\"cluster-zstream-4-20-7mzwzp\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tq69zx\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\\\",\\n  \\\"name\\\": \\\"e3ee4618-26eb-473e-867f-a23c3644e2bb\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-04-02T17:31:57.472030714Z\\\",\\n  \\\"endTime\\\": \\\"2026-04-02T17:32:55.049606818Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000732600\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-7mzwzp\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tq69zx\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\\\",\\n  \\\"name\\\": \\\"e3ee4618-26eb-473e-867f-a23c3644e2bb\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-04-02T17:31:57.472030714Z\\\",\\n  \\\"endTime\\\": \\\"2026-04-02T17:32:55.049606818Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0014af200\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadaeast/81439e1d-c310-4abd-91e5-f37963acb072\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"499\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 41853D22B093487EB33229F443FF6553 Ref B: CHI261140513025 Ref C: 2026-04-02T17:33:00Z\",\n                   ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000732640\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-7mzwzp: failed waiting for cluster=\"cluster-zstream-4-20-7mzwzp\" in resourcegroup=\"rg-zstream-upgrade-4-20-tq69zx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\",\n      \"name\": \"e3ee4618-26eb-473e-867f-a23c3644e2bb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-04-02T17:31:57.472030714Z\",\n      \"endTime\": \"2026-04-02T17:32:55.049606818Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-20-7mzwzp: failed waiting for cluster=\\\"cluster-zstream-4-20-7mzwzp\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tq69zx\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\\\",\\n  \\\"name\\\": \\\"e3ee4618-26eb-473e-867f-a23c3644e2bb\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-04-02T17:31:57.472030714Z\\\",\\n  \\\"endTime\\\": \\\"2026-04-02T17:32:55.049606818Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000732600\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-7mzwzp\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tq69zx\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\\\",\\n  \\\"name\\\": \\\"e3ee4618-26eb-473e-867f-a23c3644e2bb\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-04-02T17:31:57.472030714Z\\\",\\n  \\\"endTime\\\": \\\"2026-04-02T17:32:55.049606818Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0014af200\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadaeast/81439e1d-c310-4abd-91e5-f37963acb072\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"499\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 41853D22B093487EB33229F443FF6553 Ref B: CHI261140513025 Ref C: 2026-04-02T17:33:00Z\",\n                   ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0008d7580\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-456qg5: failed waiting for cluster=\"cluster-zstream-4-19-456qg5\" in resourcegroup=\"rg-zstream-upgrade-4-19-p762zb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\",\n      \"name\": \"78ee839b-27e8-45ab-9bef-ab361680e06a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-04-02T17:27:46.835065402Z\",\n      \"endTime\": \"2026-04-02T17:27:57.932415489Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-19-456qg5: failed waiting for cluster=\\\"cluster-zstream-4-19-456qg5\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-p762zb\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\\\",\\n  \\\"name\\\": \\\"78ee839b-27e8-45ab-9bef-ab361680e06a\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-04-02T17:27:46.835065402Z\\\",\\n  \\\"endTime\\\": \\\"2026-04-02T17:27:57.932415489Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0008d7560\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-19-456qg5\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-p762zb\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\\\",\\n  \\\"name\\\": \\\"78ee839b-27e8-45ab-9bef-ab361680e06a\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-04-02T17:27:46.835065402Z\\\",\\n  \\\"endTime\\\": \\\"2026-04-02T17:27:57.932415489Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc001317380\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Length\": [\"499\"],\n                        \"X-Ms-Request-Id\": [\n                            \"f6c45648-41ff-430b-8947-420c54510b28\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260402T172807Z:e6630a99-063f-431a-9926-65553b1b3548\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/26acb1f6-ebb4-4c9f-9e33-8c891414cd26\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: C88F82DD8BF54F1BA57C9FED21B63FDF Ref B: CHI261140514034 Ref C: 2026-04-02T17:28:07Z\",\n                        ],\n                        \"Date\": [\n                            \"Thu, 02 Apr 2026 17:28:07 GMT\",\n                        ],\n...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0008d7580\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-456qg5: failed waiting for cluster=\"cluster-zstream-4-19-456qg5\" in resourcegroup=\"rg-zstream-upgrade-4-19-p762zb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\",\n      \"name\": \"78ee839b-27e8-45ab-9bef-ab361680e06a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-04-02T17:27:46.835065402Z\",\n      \"endTime\": \"2026-04-02T17:27:57.932415489Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-19-456qg5: failed waiting for cluster=\\\"cluster-zstream-4-19-456qg5\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-p762zb\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\\\",\\n  \\\"name\\\": \\\"78ee839b-27e8-45ab-9bef-ab361680e06a\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-04-02T17:27:46.835065402Z\\\",\\n  \\\"endTime\\\": \\\"2026-04-02T17:27:57.932415489Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0008d7560\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-19-456qg5\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-p762zb\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\\\",\\n  \\\"name\\\": \\\"78ee839b-27e8-45ab-9bef-ab361680e06a\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-04-02T17:27:46.835065402Z\\\",\\n  \\\"endTime\\\": \\\"2026-04-02T17:27:57.932415489Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc001317380\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Length\": [\"499\"],\n                        \"X-Ms-Request-Id\": [\n                            \"f6c45648-41ff-430b-8947-420c54510b28\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260402T172807Z:e6630a99-063f-431a-9926-65553b1b3548\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/26acb1f6-ebb4-4c9f-9e33-8c891414cd26\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: C88F82DD8BF54F1BA57C9FED21B63FDF Ref B: CHI261140514034 Ref C: 2026-04-02T17:28:07Z\",\n                        ],\n                        \"Date\": [\n                            \"Thu, 02 Apr 2026 17:28:07 GMT\",\n                        ],\n...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0005873e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-qhj6bm: failed waiting for cluster=\"np-version-upgrade-cluster-qhj6bm\" in resourcegroup=\"rg-np-version-upgrade-qhj6bm-hp2rjl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\",\n      \"name\": \"774a88cb-0063-4a09-b407-ee6a7f59a9e6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-04-02T17:22:52.081012471Z\",\n      \"endTime\": \"2026-04-02T17:23:13.48665448Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster np-version-upgrade-cluster-qhj6bm: failed waiting for cluster=\\\"np-version-upgrade-cluster-qhj6bm\\\" in resourcegroup=\\\"rg-np-version-upgrade-qhj6bm-hp2rjl\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\\\",\\n  \\\"name\\\": \\\"774a88cb-0063-4a09-b407-ee6a7f59a9e6\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-04-02T17:22:52.081012471Z\\\",\\n  \\\"endTime\\\": \\\"2026-04-02T17:23:13.48665448Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0005873c0\u003e{\n            msg: \"failed waiting for cluster=\\\"np-version-upgrade-cluster-qhj6bm\\\" in resourcegroup=\\\"rg-np-version-upgrade-qhj6bm-hp2rjl\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\\\",\\n  \\\"name\\\": \\\"774a88cb-0063-4a09-b407-ee6a7f59a9e6\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-04-02T17:22:52.081012471Z\\\",\\n  \\\"endTime\\\": \\\"2026-04-02T17:23:13.48665448Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0010284e0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"498\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"8a74d4be-8a1d-4808-936e-22bae5b4ccd3\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 37CA783E55BC4DE4BA0F59B834B5F490 Ref B: CHI261140504062 Ref C: 2026-04-02T17:23:22Z\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"20\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260402T172322Z:8a74d4be-8a1d-4808-936e-22bae5b4ccd3\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Request-Id\": [\n                            \"c674f25c-3bae-4b96-9f1e...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0005873e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-qhj6bm: failed waiting for cluster=\"np-version-upgrade-cluster-qhj6bm\" in resourcegroup=\"rg-np-version-upgrade-qhj6bm-hp2rjl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\",\n      \"name\": \"774a88cb-0063-4a09-b407-ee6a7f59a9e6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-04-02T17:22:52.081012471Z\",\n      \"endTime\": \"2026-04-02T17:23:13.48665448Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster np-version-upgrade-cluster-qhj6bm: failed waiting for cluster=\\\"np-version-upgrade-cluster-qhj6bm\\\" in resourcegroup=\\\"rg-np-version-upgrade-qhj6bm-hp2rjl\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\\\",\\n  \\\"name\\\": \\\"774a88cb-0063-4a09-b407-ee6a7f59a9e6\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-04-02T17:22:52.081012471Z\\\",\\n  \\\"endTime\\\": \\\"2026-04-02T17:23:13.48665448Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0005873c0\u003e{\n            msg: \"failed waiting for cluster=\\\"np-version-upgrade-cluster-qhj6bm\\\" in resourcegroup=\\\"rg-np-version-upgrade-qhj6bm-hp2rjl\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\\\",\\n  \\\"name\\\": \\\"774a88cb-0063-4a09-b407-ee6a7f59a9e6\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-04-02T17:22:52.081012471Z\\\",\\n  \\\"endTime\\\": \\\"2026-04-02T17:23:13.48665448Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0010284e0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"498\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"8a74d4be-8a1d-4808-936e-22bae5b4ccd3\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 37CA783E55BC4DE4BA0F59B834B5F490 Ref B: CHI261140504062 Ref C: 2026-04-02T17:23:22Z\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"20\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260402T172322Z:8a74d4be-8a1d-4808-936e-22bae5b4ccd3\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Request-Id\": [\n                            \"c674f25c-3bae-4b96-9f1e...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039753351280726016","occurred_at":"2026-04-02T17:14:28Z","signature_id":"20e46711d3e35710a980843025c80ad4a7b4cde8409fdf3d9c4a64498fb53b05","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039753351280726016","occurred_at":"2026-04-02T17:14:28Z","signature_id":"355229726f681e41eedcc021a5f6f4da31e67f116c3a8985a272f805822bcf87","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039753351280726016","occurred_at":"2026-04-02T17:14:28Z","signature_id":"42f9dbe161009cc54c6fd91be6ef6725fe5af524a9c492cbb4f914f791198d53","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039753351280726016","occurred_at":"2026-04-02T17:14:28Z","signature_id":"5555990c760129d0559f6b21d12a328fcac98cdae5cc03808dcd0781d1280bf1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039753351280726016","occurred_at":"2026-04-02T17:14:28Z","signature_id":"9b65bed4ba440d31a2069975341012840e9a50435e2711db9b3bacbab843329f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","occurred_at":"2026-04-02T08:48:43Z","signature_id":"69b22aaa698b3b1760bc99ff5999083f5c0c44c4dcf517eb76bf27f757a7e495","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","occurred_at":"2026-04-02T08:48:43Z","signature_id":"6e970d09796e67482e58fb3616052cfbda9bac0d41b02e47fa38c4737f014b34","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","occurred_at":"2026-04-02T08:48:43Z","signature_id":"9a9415a468f807e3f900a81033319d4a094d50d5a9a6a438265cad1fac141e42","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","occurred_at":"2026-04-02T08:48:43Z","signature_id":"e12a845dc4147e276bce346cf5007864692316e60bb487e5d88e3368003a798a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","occurred_at":"2026-04-02T08:48:43Z","signature_id":"e3fcff74c9339554323e999ccbc70bc28db346f3dda20bfd1db933da9e84e721","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"2e5dafa9063ca1b7d5e22d21e90655c2cdfe7f0346c9bc2f4207e91f569aa8e6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","occurrences":6,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":6,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,0,6],"trend_range":"2026-03-27..2026-04-02 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:181]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0012b0780\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-8kqbht/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-8kqbht/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc0012f34a0\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Pragma\": [\"no-cache\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"Content-Length\": [\"107\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/1f894e75-bf9b-4f38-b36f-086c2d1ebd50\",\n                    ],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 49D6E7E0AC224A1688162351432A0445 Ref B: CH1AA2020610019 Ref C: 2026-04-02T02:19:06Z\",\n                    ],\n                    \"Date\": [\n                        \"Thu, 02 Apr 2026 02:19:06 GMT\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"4c158d49-d82a-452b-867b-1015459e1703\",\n                    ],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260402T021907Z:0ff2a06f-4992-4cf9-ae72-86b939159372\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"399\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"0ff2a06f-4992-4cf9-ae72-86b939159372\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0012b06c0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: \"manageme...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:181]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0012b0780\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-8kqbht/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-8kqbht/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc0012f34a0\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Pragma\": [\"no-cache\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"Content-Length\": [\"107\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/1f894e75-bf9b-4f38-b36f-086c2d1ebd50\",\n                    ],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 49D6E7E0AC224A1688162351432A0445 Ref B: CH1AA2020610019 Ref C: 2026-04-02T02:19:06Z\",\n                    ],\n                    \"Date\": [\n                        \"Thu, 02 Apr 2026 02:19:06 GMT\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"4c158d49-d82a-452b-867b-1015459e1703\",\n                    ],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260402T021907Z:0ff2a06f-4992-4cf9-ae72-86b939159372\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"399\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"0ff2a06f-4992-4cf9-ae72-86b939159372\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0012b06c0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: \"manageme...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:147]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0008c23a0\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-c4ph7r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-pw4hx4/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-c4ph7r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-pw4hx4/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc00031b110\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260402T022005Z:bae2f98c-c749-481d-9054-9921e081945e\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/62d919b4-4225-4967-9df6-08b0b5351023\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Date\": [\n                        \"Thu, 02 Apr 2026 02:20:05 GMT\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Request-Id\": [\n                        \"4c5f9568-bb9b-40c3-81ad-0ca630f99d24\",\n                    ],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 2552FC52616C4D5BB46364A7B7C0692E Ref B: CH1AA2020617017 Ref C: 2026-04-02T02:20:02Z\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"2628\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"Content-Length\": [\"107\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"bae2f98c-c749-481d-9054-9921e081945e\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0008c21e0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                  ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:147]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0008c23a0\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-c4ph7r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-pw4hx4/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-c4ph7r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-pw4hx4/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc00031b110\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260402T022005Z:bae2f98c-c749-481d-9054-9921e081945e\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/62d919b4-4225-4967-9df6-08b0b5351023\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Date\": [\n                        \"Thu, 02 Apr 2026 02:20:05 GMT\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Request-Id\": [\n                        \"4c5f9568-bb9b-40c3-81ad-0ca630f99d24\",\n                    ],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 2552FC52616C4D5BB46364A7B7C0692E Ref B: CH1AA2020617017 Ref C: 2026-04-02T02:20:02Z\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"2628\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"Content-Length\": [\"107\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"bae2f98c-c749-481d-9054-9921e081945e\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0008c21e0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                  ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:118]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001418640\u003e: \n    failed creating external auth \"ea-list\" in resourcegroup=\"ea-list-sqv9wn\" for cluster=\"ea-list4lvwkr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-sqv9wn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-list4lvwkr/externalAuths/ea-list\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed creating external auth \\\"ea-list\\\" in resourcegroup=\\\"ea-list-sqv9wn\\\" for cluster=\\\"ea-list4lvwkr\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-sqv9wn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-list4lvwkr/externalAuths/ea-list\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc001366990\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Request-Id\": [\n                        \"f4e61ff2-b02c-4f78-930a-516f5367173c\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"Date\": [\n                        \"Thu, 02 Apr 2026 02:10:05 GMT\",\n                    ],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 04AE8E5AF86B478685A5B528872AB82B Ref B: CHI261140505062 Ref C: 2026-04-02T02:09:59Z\",\n                    ],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/786cdfbd-1a09-42c1-be7d-896a3083330d\",\n                    ],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"4be39819-497a-401e-a7b1-442d6a3bb89a\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"7051\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260402T021006Z:4be39819-497a-401e-a7b1-442d6a3bb89a\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Type\": [\"application/json\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc001418500\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"PUT\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n             ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:118]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001418640\u003e: \n    failed creating external auth \"ea-list\" in resourcegroup=\"ea-list-sqv9wn\" for cluster=\"ea-list4lvwkr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-sqv9wn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-list4lvwkr/externalAuths/ea-list\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed creating external auth \\\"ea-list\\\" in resourcegroup=\\\"ea-list-sqv9wn\\\" for cluster=\\\"ea-list4lvwkr\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-sqv9wn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-list4lvwkr/externalAuths/ea-list\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc001366990\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Request-Id\": [\n                        \"f4e61ff2-b02c-4f78-930a-516f5367173c\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"Date\": [\n                        \"Thu, 02 Apr 2026 02:10:05 GMT\",\n                    ],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 04AE8E5AF86B478685A5B528872AB82B Ref B: CHI261140505062 Ref C: 2026-04-02T02:09:59Z\",\n                    ],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/786cdfbd-1a09-42c1-be7d-896a3083330d\",\n                    ],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"4be39819-497a-401e-a7b1-442d6a3bb89a\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"7051\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260402T021006Z:4be39819-497a-401e-a7b1-442d6a3bb89a\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Type\": [\"application/json\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc001418500\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"PUT\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n             ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","signature_id":"4d26299f52b0b8986f5cb45926b1e2da9613d8a375ee0a382f06b7e2c7a01945","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","signature_id":"6eacec424ad1161d7aacbab03f8a93ea1c58263a875329c1260729344930c215","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","signature_id":"7a56671f4aff56ecc03b701290aceff3662c37535980cb96bf3f4b76002fa5d3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","signature_id":"a065c8647285da4ae861666481d317b07db2efdc79edc92e410f5b8bacb3f1ae","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","signature_id":"d3286a20e487ac27f27dee83ac29804121c9b49a8c8b8ea4d003f723f9be0d69","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","signature_id":"efed37f596e7501dc50ed6fc82727bd5d2db1dd41b58a065c7155228aa4d0495","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"47c4fe17c8b40b438461015820e355e731ca2f90428cea3ac600911bb9a03df7","failure_pattern":"ERROR CODE: ServiceUnavailable; detail message Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ServiceUnavailable","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":3,"after_last_push_count":11,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-03-27..2026-04-02 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-02","2026-03-23"],"prior_runs_affected":4,"prior_last_seen_at":"2026-03-25T19:15:18Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00075c1e0\u003e: \n    failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-wvg7xj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-wvg7xj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: ServiceUnavailable\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ServiceUnavailable\",\n        \"message\": \"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster patch-name-cluster: failed starting cluster creation \\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-wvg7xj\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-wvg7xj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ServiceUnavailable\\\",\\n    \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00075c1c0\u003e{\n            msg: \"failed starting cluster creation \\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-wvg7xj\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-wvg7xj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ServiceUnavailable\\\",\\n    \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00004d2f0\u003e{\n                ErrorCode: \"ServiceUnavailable\",\n                StatusCode: 503,\n                RawResponse: {\n                    Status: \"503 Service Unavailable\",\n                    StatusCode: 503,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"2878\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"0b9741c0-9777-4781-b4fb-bf1afce41505\",\n                        ],\n                        \"Retry-After\": [\"59\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                        \"Date\": [\n                            \"Thu, 02 Apr 2026 08:57:47 GMT\",\n                        ],\n                        \"Content-Length\": [\"257\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Error-Code\": [\"ServiceUnavailable\"],\n                        \"X-Ms-Request-Id\": [\n                            \"87531355-f28f-4c12-bf8b-c7f8a31db6ce\",\n                        ],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/7ad18b62-8874-4d58-81d6-01921cd42fc7\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"UKSOUTH:20260402T085747Z:0b9741c0-9777-4781-b4fb-bf1afce41505\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00075c1e0\u003e: \n    failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-wvg7xj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-wvg7xj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: ServiceUnavailable\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ServiceUnavailable\",\n        \"message\": \"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster patch-name-cluster: failed starting cluster creation \\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-wvg7xj\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-wvg7xj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ServiceUnavailable\\\",\\n    \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00075c1c0\u003e{\n            msg: \"failed starting cluster creation \\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-wvg7xj\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-wvg7xj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ServiceUnavailable\\\",\\n    \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00004d2f0\u003e{\n                ErrorCode: \"ServiceUnavailable\",\n                StatusCode: 503,\n                RawResponse: {\n                    Status: \"503 Service Unavailable\",\n                    StatusCode: 503,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"2878\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"0b9741c0-9777-4781-b4fb-bf1afce41505\",\n                        ],\n                        \"Retry-After\": [\"59\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                        \"Date\": [\n                            \"Thu, 02 Apr 2026 08:57:47 GMT\",\n                        ],\n                        \"Content-Length\": [\"257\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Error-Code\": [\"ServiceUnavailable\"],\n                        \"X-Ms-Request-Id\": [\n                            \"87531355-f28f-4c12-bf8b-c7f8a31db6ce\",\n                        ],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/7ad18b62-8874-4d58-81d6-01921cd42fc7\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"UKSOUTH:20260402T085747Z:0b9741c0-9777-4781-b4fb-bf1afce41505\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:123]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0004e6700\u003e: \n    failed starting cluster creation \"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-p6dr85\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-p6dr85/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster2\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: ServiceUnavailable\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ServiceUnavailable\",\n        \"message\": \"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed starting cluster creation \\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-p6dr85\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-p6dr85/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster2\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ServiceUnavailable\\\",\\n    \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc001402630\u003e{\n            ErrorCode: \"ServiceUnavailable\",\n            StatusCode: 503,\n            RawResponse: {\n                Status: \"503 Service Unavailable\",\n                StatusCode: 503,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Request-Id\": [\n                        \"981e3b54-276e-45f0-a48f-b05f015c79dc\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"8461\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/d750dcff-9cd4-44ec-8a65-6f1b69e57a9d\",\n                    ],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"45bdda71-4b5b-4b9b-9fd7-4df364bf7808\",\n                    ],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260402T085811Z:45bdda71-4b5b-4b9b-9fd7-4df364bf7808\",\n                    ],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Length\": [\"257\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Error-Code\": [\"ServiceUnavailable\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: C36632182EBE4B8EB52F9106BF8C5BB2 Ref B: CHI261140502029 Ref C: 2026-04-02T08:58:02Z\",\n                    ],\n                    \"Retry-After\": [\"59\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Date\": [\n                        \"Thu, 02 Apr 2026 08:58:11 GMT\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0004e6480\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 83, 101, 114, 118, 105, 99, 101, 85, 110, 97, 118, 97, 105, 108, 97, 98, 108, 101, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 108, 117, 115, 116, 101, 114, 32, 105, 110, 115, 116, 97, 108, 108, 97, 116, 105, 111, 110, 32, 104, 97, 115, 32, 102, 97, 105, 108, 101, 100, 32, 100, 117, 101, 32, 116, 111, 32, 117, 110, 97, 118, 97, 105, 108, 97, 98, 108, 101, 32, 99, 97, 112, 97, 99, 105, 116, 121, 46, 32, 87, 104, 105, 108, 101, 32, 119, 101, 32, 97, 114, 101, 32, 98, 114, 105, 110, 103, 105, 110, 103, 32, 111, 110, 32, 97, 100, 100, 105, 116, 105, 111, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:123]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0004e6700\u003e: \n    failed starting cluster creation \"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-p6dr85\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-p6dr85/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster2\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: ServiceUnavailable\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ServiceUnavailable\",\n        \"message\": \"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed starting cluster creation \\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-p6dr85\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-p6dr85/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster2\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ServiceUnavailable\\\",\\n    \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc001402630\u003e{\n            ErrorCode: \"ServiceUnavailable\",\n            StatusCode: 503,\n            RawResponse: {\n                Status: \"503 Service Unavailable\",\n                StatusCode: 503,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Request-Id\": [\n                        \"981e3b54-276e-45f0-a48f-b05f015c79dc\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"8461\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/d750dcff-9cd4-44ec-8a65-6f1b69e57a9d\",\n                    ],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"45bdda71-4b5b-4b9b-9fd7-4df364bf7808\",\n                    ],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260402T085811Z:45bdda71-4b5b-4b9b-9fd7-4df364bf7808\",\n                    ],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Length\": [\"257\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Error-Code\": [\"ServiceUnavailable\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: C36632182EBE4B8EB52F9106BF8C5BB2 Ref B: CHI261140502029 Ref C: 2026-04-02T08:58:02Z\",\n                    ],\n                    \"Retry-After\": [\"59\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Date\": [\n                        \"Thu, 02 Apr 2026 08:58:11 GMT\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0004e6480\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 83, 101, 114, 118, 105, 99, 101, 85, 110, 97, 118, 97, 105, 108, 97, 98, 108, 101, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 108, 117, 115, 116, 101, 114, 32, 105, 110, 115, 116, 97, 108, 108, 97, 116, 105, 111, 110, 32, 104, 97, 115, 32, 102, 97, 105, 108, 101, 100, 32, 100, 117, 101, 32, 116, 111, 32, 117, 110, 97, 118, 97, 105, 108, 97, 98, 108, 101, 32, 99, 97, 112, 97, 99, 105, 116, 121, 46, 32, 87, 104, 105, 108, 101, 32, 119, 101, 32, 97, 114, 101, 32, 98, 114, 105, 110, 103, 105, 110, 103, 32, 111, 110, 32, 97, 100, 100, 105, 116, 105, 111, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000880260\u003e: \n    failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-6tff8z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-6tff8z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: ServiceUnavailable\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ServiceUnavailable\",\n        \"message\": \"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster basic-cluster: failed starting cluster creation \\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-6tff8z\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-6tff8z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ServiceUnavailable\\\",\\n    \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000880200\u003e{\n            msg: \"failed starting cluster creation \\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-6tff8z\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-6tff8z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ServiceUnavailable\\\",\\n    \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0012ca1b0\u003e{\n                ErrorCode: \"ServiceUnavailable\",\n                StatusCode: 503,\n                RawResponse: {\n                    Status: \"503 Service Unavailable\",\n                    StatusCode: 503,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Request-Id\": [\n                            \"2a87bdad-8340-4cda-a740-2d7cc278c733\",\n                        ],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"294bd1e5-7abf-4f05-a3c2-931fa8a72c4e\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"UKSOUTH:20260402T085750Z:294bd1e5-7abf-4f05-a3c2-931fa8a72c4e\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"Date\": [\n                            \"Thu, 02 Apr 2026 08:57:50 GMT\",\n                        ],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"2649\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/cc9e8570-fcc2-4ba9-9d49-08bb67825537\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Content-Length\": [\"257\"],\n                        \"Retry-After\": [\"59\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Ms-Fail...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000880260\u003e: \n    failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-6tff8z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-6tff8z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: ServiceUnavailable\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ServiceUnavailable\",\n        \"message\": \"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster basic-cluster: failed starting cluster creation \\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-6tff8z\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-6tff8z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ServiceUnavailable\\\",\\n    \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000880200\u003e{\n            msg: \"failed starting cluster creation \\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-6tff8z\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-6tff8z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ServiceUnavailable\\\",\\n    \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0012ca1b0\u003e{\n                ErrorCode: \"ServiceUnavailable\",\n                StatusCode: 503,\n                RawResponse: {\n                    Status: \"503 Service Unavailable\",\n                    StatusCode: 503,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Request-Id\": [\n                            \"2a87bdad-8340-4cda-a740-2d7cc278c733\",\n                        ],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"294bd1e5-7abf-4f05-a3c2-931fa8a72c4e\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"UKSOUTH:20260402T085750Z:294bd1e5-7abf-4f05-a3c2-931fa8a72c4e\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"Date\": [\n                            \"Thu, 02 Apr 2026 08:57:50 GMT\",\n                        ],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"2649\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/cc9e8570-fcc2-4ba9-9d49-08bb67825537\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Content-Length\": [\"257\"],\n                        \"Retry-After\": [\"59\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Ms-Fail...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","occurred_at":"2026-04-02T08:48:43Z","signature_id":"3a2e25e9d1890483fd272049272851235e7e4ec464629137b0a1a3d7b8980e7f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","occurred_at":"2026-04-02T08:48:43Z","signature_id":"cacb304bc7bd5ba62586475b5881017dd5ae0a6ac7b13d88e7aa4f249a8ed2ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","occurred_at":"2026-04-02T08:48:43Z","signature_id":"fd76359852bfe92722186a57e697c3c69f783a8157d4dcae452c9c7c11c50934","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"e17b72ff49490f6e1940fa5deb5057f7012ebd4b2cee2f2d2e8b8d47e6784bb9","failure_pattern":"ERROR CODE: LockContention; detail message Failed to acquire lock: timed out; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LockContention","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":2,"after_last_push_count":11,"trend_counts":[0,0,0,5,0,0,2],"trend_range":"2026-03-27..2026-04-02 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-02","2026-03-09"],"prior_runs_affected":6,"prior_last_seen_at":"2026-03-13T06:56:30Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009f2840\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-k2qhj7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: LockContention\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LockContention\",\n        \"message\": \"Failed to acquire lock: timed out\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-k2qhj7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"LockContention\\\",\\n    \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc00159e990\u003e{\n            ErrorCode: \"LockContention\",\n            StatusCode: 503,\n            RawResponse: {\n                Status: \"503 Service Unavailable\",\n                StatusCode: 503,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/1c5d69eb-b218-43c3-89e1-797546221c53\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Date\": [\n                        \"Thu, 02 Apr 2026 02:10:08 GMT\",\n                    ],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Error-Code\": [\"LockContention\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260402T021008Z:afdf4447-88c0-4010-85eb-2250eadd30fe\",\n                    ],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 1EC287F8926644E099B85CCB3B5E9590 Ref B: CHI261140504042 Ref C: 2026-04-02T02:09:58Z\",\n                    ],\n                    \"Content-Length\": [\"186\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Request-Id\": [\n                        \"2b953333-0bff-468f-ad05-d66baee5ebf5\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"10006\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"afdf4447-88c0-4010-85eb-2250eadd30fe\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Retry-After\": [\"10\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0009f2700\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 76, 111, 99, 107, 67, 111, 110, 116, 101, 110, 116, 105, 111, 110, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 32, 116, 111, 32, 97, 99, 113, 117, 105, 114, 101, 32, 108, 111, 99, 107, 58, 32, 116, 105, 109, 101, 100, 32, 111, 117, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 57, 51, 57, 57, 50, 56, 49, 45, 48, 48, 97, 50, 45, 52, 98, 51, 57, 45, 98, 98, 51, 100, 45, 98, 50, 54, 52, 53, 98, 98, 98, 100, 98, 57, 51, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                Conte...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009f2840\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-k2qhj7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: LockContention\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LockContention\",\n        \"message\": \"Failed to acquire lock: timed out\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-k2qhj7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"LockContention\\\",\\n    \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc00159e990\u003e{\n            ErrorCode: \"LockContention\",\n            StatusCode: 503,\n            RawResponse: {\n                Status: \"503 Service Unavailable\",\n                StatusCode: 503,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/1c5d69eb-b218-43c3-89e1-797546221c53\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Date\": [\n                        \"Thu, 02 Apr 2026 02:10:08 GMT\",\n                    ],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Error-Code\": [\"LockContention\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260402T021008Z:afdf4447-88c0-4010-85eb-2250eadd30fe\",\n                    ],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 1EC287F8926644E099B85CCB3B5E9590 Ref B: CHI261140504042 Ref C: 2026-04-02T02:09:58Z\",\n                    ],\n                    \"Content-Length\": [\"186\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Request-Id\": [\n                        \"2b953333-0bff-468f-ad05-d66baee5ebf5\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"10006\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"afdf4447-88c0-4010-85eb-2250eadd30fe\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Retry-After\": [\"10\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0009f2700\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 76, 111, 99, 107, 67, 111, 110, 116, 101, 110, 116, 105, 111, 110, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 32, 116, 111, 32, 97, 99, 113, 117, 105, 114, 101, 32, 108, 111, 99, 107, 58, 32, 116, 105, 109, 101, 100, 32, 111, 117, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 57, 51, 57, 57, 50, 56, 49, 45, 48, 48, 97, 50, 45, 52, 98, 51, 57, 45, 98, 98, 51, 100, 45, 98, 50, 54, 52, 53, 98, 98, 98, 100, 98, 57, 51, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                Conte...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:103]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00142ee20\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-4qqv6m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: LockContention\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LockContention\",\n        \"message\": \"Failed to acquire lock: timed out\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-4qqv6m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"LockContention\\\",\\n    \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc00104d530\u003e{\n            ErrorCode: \"LockContention\",\n            StatusCode: 503,\n            RawResponse: {\n                Status: \"503 Service Unavailable\",\n                StatusCode: 503,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/0949565f-2116-4c2d-ac42-f874e16826f1\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260402T021009Z:c60ff00c-0498-471e-83f9-643c3970b3df\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 84A2B7061A1D427CAC14B9844929E76E Ref B: CHI261140502054 Ref C: 2026-04-02T02:09:58Z\",\n                    ],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Length\": [\"186\"],\n                    \"Retry-After\": [\"10\"],\n                    \"X-Ms-Error-Code\": [\"LockContention\"],\n                    \"X-Ms-Request-Id\": [\n                        \"342b364f-60a1-493c-a980-5f75e6f1c77f\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"10010\"],\n                    \"Date\": [\n                        \"Thu, 02 Apr 2026 02:10:08 GMT\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"c60ff00c-0498-471e-83f9-643c3970b3df\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc00142ed60\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 76, 111, 99, 107, 67, 111, 110, 116, 101, 110, 116, 105, 111, 110, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 32, 116, 111, 32, 97, 99, 113, 117, 105, 114, 101, 32, 108, 111, 99, 107, 58, 32, 116, 105, 109, 101, 100, 32, 111, 117, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 57, 51, 57, 57, 50, 56, 49, 45, 48, 48, 97, 50, 45, 52, 98, 51, 57, 45, 98, 98, 51, 100, 45, 98, 50, 54, 52, 53, 98, 98, 98, 100, 98, 57, 51, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                Co...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:103]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00142ee20\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-4qqv6m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: LockContention\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LockContention\",\n        \"message\": \"Failed to acquire lock: timed out\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-4qqv6m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"LockContention\\\",\\n    \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc00104d530\u003e{\n            ErrorCode: \"LockContention\",\n            StatusCode: 503,\n            RawResponse: {\n                Status: \"503 Service Unavailable\",\n                StatusCode: 503,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/0949565f-2116-4c2d-ac42-f874e16826f1\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260402T021009Z:c60ff00c-0498-471e-83f9-643c3970b3df\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 84A2B7061A1D427CAC14B9844929E76E Ref B: CHI261140502054 Ref C: 2026-04-02T02:09:58Z\",\n                    ],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Length\": [\"186\"],\n                    \"Retry-After\": [\"10\"],\n                    \"X-Ms-Error-Code\": [\"LockContention\"],\n                    \"X-Ms-Request-Id\": [\n                        \"342b364f-60a1-493c-a980-5f75e6f1c77f\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"10010\"],\n                    \"Date\": [\n                        \"Thu, 02 Apr 2026 02:10:08 GMT\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"c60ff00c-0498-471e-83f9-643c3970b3df\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc00142ed60\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 76, 111, 99, 107, 67, 111, 110, 116, 101, 110, 116, 105, 111, 110, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 32, 116, 111, 32, 97, 99, 113, 117, 105, 114, 101, 32, 108, 111, 99, 107, 58, 32, 116, 105, 109, 101, 100, 32, 111, 117, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 57, 51, 57, 57, 50, 56, 49, 45, 48, 48, 97, 50, 45, 52, 98, 51, 57, 45, 98, 98, 51, 100, 45, 98, 50, 54, 52, 53, 98, 98, 98, 100, 98, 57, 51, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                Co...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","signature_id":"61fec9e174cbe2103641b0ca2d00e068a13dee8925feb67a852f2d37175153ed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","signature_id":"ad02bdcbba8a2cd55fe28294470d643c39c011c3cd16fba715a90713c1dd2990","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}