{"meta":{"start_date":"2026-03-30","end_date":"2026-04-05","start_at":"2026-03-30T00:00:00Z","end_at":"2026-04-06T00:00:00Z","timezone":"UTC","generated_at":"2026-08-27T11:48:14Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":20,"failed_runs":15,"raw_occurrences":108,"matched_occurrences":108,"runs_affected":15},"rows":[{"environment":"stg","failure_pattern_id":"61f2f606bce557ba1356502d9d712c9f0ba12bc1f568bcc162a9382b70354d2b","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-7bwpw7\" in resourcegroup=\"admin-credential-lifecycle-test-ms8t98\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-7bwpw7 in resource group admin-credential-lifecycle-test-ms8t98, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":63,"runs_affected":7,"failed_runs":7,"run_impact_percent":35,"anchor_occurrences":63,"after_last_push_count":65,"trend_counts":[0,0,0,9,25,11,7,11],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-23"],"prior_runs_affected":8,"prior_last_seen_at":"2026-03-23T02:00:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:101]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0017689c0\u003e: \n    failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-j8897x\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-j8897x, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-np-labels-taints\\\" in resourcegroup=\\\"rg-np-labels-taints-j8897x\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-j8897x, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0009f37b0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-j8897x\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:101]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0017689c0\u003e: \n    failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-j8897x\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-j8897x, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-np-labels-taints\\\" in resourcegroup=\\\"rg-np-labels-taints-j8897x\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-j8897x, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0009f37b0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-np-labels-taints in resource group rg-np-labels-taints-j8897x\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00168c360\u003e: \n    failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-gcpcqw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-gcpcqw, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-gcpcqw\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-gcpcqw, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000707550\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-gcpcqw\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00168c360\u003e: \n    failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-gcpcqw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-gcpcqw, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-gcpcqw\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-gcpcqw, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000707550\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-gcpcqw\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0001ff650\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-21-nvk8p8\" in resourcegroup=\"rg-zstream-upgrade-4-21-2rn9fm\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-nvk8p8 in resource group rg-zstream-upgrade-4-21-2rn9fm, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-nvk8p8\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-2rn9fm\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-nvk8p8 in resource group rg-zstream-upgrade-4-21-2rn9fm, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0006c5000\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-nvk8p8 in resource group rg-zstream-upgrade-4-21-2rn9fm\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0001ff650\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-21-nvk8p8\" in resourcegroup=\"rg-zstream-upgrade-4-21-2rn9fm\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-nvk8p8 in resource group rg-zstream-upgrade-4-21-2rn9fm, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-21-nvk8p8\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-2rn9fm\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-nvk8p8 in resource group rg-zstream-upgrade-4-21-2rn9fm, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0006c5000\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-21-nvk8p8 in resource group rg-zstream-upgrade-4-21-2rn9fm\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040610572591960064","occurred_at":"2026-04-05T02:00:47Z","signature_id":"160343a4706949b411412383554edf65feacb47e531c85f36792e09ea81bfd61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040610572591960064","occurred_at":"2026-04-05T02:00:47Z","signature_id":"166c481e05dfa45f474630a2ad6506207858e1c0fe1ff6841f9570a3b2b67713","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040610572591960064","occurred_at":"2026-04-05T02:00:47Z","signature_id":"169da7ba0de579028fbee4be3db42421bc5d4bdc651957b5e2b43be3c542afa8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040610572591960064","occurred_at":"2026-04-05T02:00:47Z","signature_id":"37cc6d105a0f804ec42a46d399a6d1c6a9ae826865041827486aec0e02b48123","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040610572591960064","occurred_at":"2026-04-05T02:00:47Z","signature_id":"6e774cd230f1e30ffc4158e39821deb7e24175b5bbe89ce7fa503eab80fc33fb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040610572591960064","occurred_at":"2026-04-05T02:00:47Z","signature_id":"85bc15f64c40a713773c9873c6aac805700206d62547e1963acde98d91833cc2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040610572591960064","occurred_at":"2026-04-05T02:00:47Z","signature_id":"ab96e4d11d001ab3759c72c6cea80f0729a9a5df6ba635cf9b60f47680b8c0df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040610572591960064","occurred_at":"2026-04-05T02:00:47Z","signature_id":"b8f06d36acc683e4d38997f5da336bea819cf882f01efa04e35f9621435644fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040610572591960064","occurred_at":"2026-04-05T02:00:47Z","signature_id":"b9bf2b5d83a17242847e8a00814cf1f89bef7413d8a9f6d2bbccc575a3ed6bc0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040610572591960064","occurred_at":"2026-04-05T02:00:47Z","signature_id":"dfab9f6f9411953429b4c4e4727775ea2035a509097ae09169e76b0cf1773401","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040610572591960064","occurred_at":"2026-04-05T02:00:47Z","signature_id":"f9f9e4c56d4385f8a63bbb59c4dbb438f1bca107732e56e7d9bf77bc84a64b90","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040248180829851648","occurred_at":"2026-04-04T02:00:46Z","signature_id":"33300053dbad9e28a7270a95a3547fb91c65af7d71efea9d3908c366d7c9e2f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040248180829851648","occurred_at":"2026-04-04T02:00:46Z","signature_id":"425163d3b07eeea206817fea9114ce231c7affa81d68cb7bdd6b0d630517c767","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040248180829851648","occurred_at":"2026-04-04T02:00:46Z","signature_id":"5a24aa7b5b52f72c55b7b10585864266c0ba8924e164a4fdd00dbfb199b8c224","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040248180829851648","occurred_at":"2026-04-04T02:00:46Z","signature_id":"ab9c2f45c3c3c691585d16285aa143f923279bb69be1ade0d1f5a95650ca968c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040248180829851648","occurred_at":"2026-04-04T02:00:46Z","signature_id":"c6f683aea5f50d72108b2d70a37325bdad079fb0b15160489207e7d2a5764760","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040248180829851648","occurred_at":"2026-04-04T02:00:46Z","signature_id":"d2c18c1ce7acab0b289b6521ca0f196b8a3fe81360e8f107f1880ea6a6437f48","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040248180829851648","occurred_at":"2026-04-04T02:00:46Z","signature_id":"ebf05020222bb7411531b4209113a1628b22b3b8eda5ed2a04777e3327620d73","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039885780997902336","occurred_at":"2026-04-03T02:00:44Z","signature_id":"0ceb29be6fa375ea57e4173558fc494993fd145bdc27922799cf8834dac7ef04","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039885780997902336","occurred_at":"2026-04-03T02:00:44Z","signature_id":"0ddcf9b7a9b560237a63664c822dae6230a3f2c38022f974bf36375d0c62ac26","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039885780997902336","occurred_at":"2026-04-03T02:00:44Z","signature_id":"4c9396631e94913d3d9769b506d66c84d227fe645c5fff2f39143553dde2fb48","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039885780997902336","occurred_at":"2026-04-03T02:00:44Z","signature_id":"4df295d4bcf1c3a25bcb9d4313ec95cb3b70e10b039816a475c8e2dcf4ebea5c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039885780997902336","occurred_at":"2026-04-03T02:00:44Z","signature_id":"673ca9f22262ba954e6c619d8c486719adc7efece6dd4d226407adc1ca8ecfff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039885780997902336","occurred_at":"2026-04-03T02:00:44Z","signature_id":"7449efc74942e243b421e2fdd0dbcbefcce7fc9ff197ee33536477954a91c289","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039885780997902336","occurred_at":"2026-04-03T02:00:44Z","signature_id":"81f53d8f2067544bad508db752956128cd306cb509b0927eb9dc34c3f9b3ec8c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039885780997902336","occurred_at":"2026-04-03T02:00:44Z","signature_id":"85260ba9d0c809a8bcbb7c3cd97a09b960a4f3e600d7b427acfe47daa64c59ee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039885780997902336","occurred_at":"2026-04-03T02:00:44Z","signature_id":"a18b47cc9e3230ad4dc34a04629d5d3956803e4f01afa9621198604b395ce997","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039885780997902336","occurred_at":"2026-04-03T02:00:44Z","signature_id":"d5942a894dd48c6de5acbd3fcb6d83380e285c8a3511f8e3b9f70047aec3bd29","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039885780997902336","occurred_at":"2026-04-03T02:00:44Z","signature_id":"d6bddf186b5543b24cdc4aa069be3cac9f30f503b3f461054cd89c61ccc3354f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039846445451841536","occurred_at":"2026-04-02T23:24:25Z","signature_id":"425cb192426cfae26c27c30de57f81c15b90dc678940b4bca02416c7cb2beaed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039846445451841536","occurred_at":"2026-04-02T23:24:25Z","signature_id":"48700371ca5e3ea6f36170d783d442d058d1cb1424257da18a3fc2d24e97609f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039846445451841536","occurred_at":"2026-04-02T23:24:25Z","signature_id":"70daa82f0e74a2a68d0f694e0626bdd2ee7f59b84ad17694935a98cf2693387c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039846445451841536","occurred_at":"2026-04-02T23:24:25Z","signature_id":"925d5b5e8d1cb7062ad7f97148097b4bbbd90c0a007478dcd253302060efce42","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039846445451841536","occurred_at":"2026-04-02T23:24:25Z","signature_id":"abbe9ccf77f58b9dcecdebc516030b33899c9d00e6c69935ca042cc483f352bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039846445451841536","occurred_at":"2026-04-02T23:24:25Z","signature_id":"daa01e0a071bb23981629322b51a8a838a0a20c385d7cccf1bda9ff5fe01bb42","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039846445451841536","occurred_at":"2026-04-02T23:24:25Z","signature_id":"f62f0cea8a0efc666a80f05794478f17dc1e76b6c132ca2c22eb98b5fbbbd8d1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"1c8fe61fbd43db9169aaa7fbf69609a8728f6d8509220f9a92d6f861ef91f8c0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"3a274d8c90c6db3b46a4ec71e46a64843c4377cb4ead33281af23bd84b9ab97d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"473710e31048fab0665756ba98c3a4020c0308701ea4c9a9babf37c2b2ab4f39","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"4c6fe9b34a98463c74e064da1997e5316b414aefa31a08707da1116847153d72","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"5e59c002f97b94a2b5076e852a21ebe711b44b0942e4ecc5862f23348be56977","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"61f89257fbd679c1f5a3f14ecb2cfdc110c59434b5cb1139ebf64baa58624853","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"6b5479644ee17536d3c82e89ebbb26e7571bb914b168ab9d74f281baf41fd90c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"807f4398a1b973a767b11bdff3ba6ffe1f98c0444aa1aecbf515704d79fa35f2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"87f712f0082578aceb2139869be1b2173936efbf987880da72d2af8ef95f3557","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"96e939a1dc8a1464c8152e2dcfbf00623eaef4723e51466619b88ddbd5291584","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"a77aba6e2f5896ef8dd63e5789b9c1381fb9c0ef40adec0f73550907fb8efca7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"bd8c64743544c200535996997a509a9d5a3ba8e598a513d39001e796b305da6a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"c6f66b20a06674c7be9ba89a2d96b7d1acc8a1ac27f90ece2f2477e63df258d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039805076003885056","occurred_at":"2026-04-02T20:40:02Z","signature_id":"f3edf87053a3ddfa94c5d07ba8914a620291e1ca95012ba77d27456596d7a2f7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","signature_id":"8a5c03c98dbcaa4cc970195ad454045945fe07e39d81c73216bb358fdf6301fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","signature_id":"90bf4c501cbcfdc4b92b5c6d62b79255bb6cb4ec6dd2de919de62549dec49677","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","signature_id":"ae1b83127fad3f1bb11977b594bb276b77277cef01ce027657dee63ae31ce624","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","signature_id":"afe25b897c006e3672b59622d300b23541d7af3aa6c8e6237eeb004b4e3541ee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039160988686618624","occurred_at":"2026-04-01T02:00:40Z","signature_id":"99c415edd6b95eec8761b3671d6423a3a146ba7c00178112a04e67159626c816","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039160988686618624","occurred_at":"2026-04-01T02:00:40Z","signature_id":"9b101c4a736d523bb15934eea086cc5ca2341cec5f5bfab68fe667ee172a4857","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039160988686618624","occurred_at":"2026-04-01T02:00:40Z","signature_id":"b3da81362b9698cffdac12e8ad5fd90a828a899d06d074781eeb6cefeafa8ec4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039160988686618624","occurred_at":"2026-04-01T02:00:40Z","signature_id":"b8e8fab0cce09ad90ef2251627883aa3bf86e27e2c20dbe95e2cea30446f507a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039160988686618624","occurred_at":"2026-04-01T02:00:40Z","signature_id":"bbad72fda5e343074c8c3e32f9cba154886471d76368cdd557ef9784681db104","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039160988686618624","occurred_at":"2026-04-01T02:00:40Z","signature_id":"c3b9140d3fdfc6f04844c1305bbffd8c10e94819b3a4ebe810b753e959596bf0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039160988686618624","occurred_at":"2026-04-01T02:00:40Z","signature_id":"d18e01fea11974ed0eba492770e2f8a7d174f753a3b28796e1137234faeb8272","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039160988686618624","occurred_at":"2026-04-01T02:00:40Z","signature_id":"f88f46b79d01481827ca3fd0763047f1a5e1f65a35470b8d19a7ff3d98a3330b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039160988686618624","occurred_at":"2026-04-01T02:00:40Z","signature_id":"fcf6f271f147ca0d27070d73d4c1265abf1b5b47346a91a590dc07c731bfb2f6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"2e5dafa9063ca1b7d5e22d21e90655c2cdfe7f0346c9bc2f4207e91f569aa8e6","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":11,"runs_affected":4,"failed_runs":4,"run_impact_percent":20,"anchor_occurrences":11,"after_last_push_count":11,"trend_counts":[0,0,0,0,6,3,1,1],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:155]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00089c2a0\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-cwknbt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-cwknbt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc001702360\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Request-Id\": [\n                        \"69968efa-5e53-4e34-a908-436feea7199a\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"3908\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/879d1d49-767f-4337-a84e-8f1c585bec0a\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260405T024147Z:f9b47596-ceaf-4492-94b7-7e7939ee00f1\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 8142C57A62E143F1BD408CF8DCE6F3B4 Ref B: CHI261140502031 Ref C: 2026-04-05T02:41:43Z\",\n                    ],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"f9b47596-ceaf-4492-94b7-7e7939ee00f1\",\n                    ],\n                    \"Date\": [\n                        \"Sun, 05 Apr 2026 02:41:46 GMT\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Content-Type\": [\"application/json\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc00089c1e0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: \"...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:155]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00089c2a0\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-cwknbt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-cwknbt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc001702360\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Request-Id\": [\n                        \"69968efa-5e53-4e34-a908-436feea7199a\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"3908\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/879d1d49-767f-4337-a84e-8f1c585bec0a\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260405T024147Z:f9b47596-ceaf-4492-94b7-7e7939ee00f1\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 8142C57A62E143F1BD408CF8DCE6F3B4 Ref B: CHI261140502031 Ref C: 2026-04-05T02:41:43Z\",\n                    ],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"f9b47596-ceaf-4492-94b7-7e7939ee00f1\",\n                    ],\n                    \"Date\": [\n                        \"Sun, 05 Apr 2026 02:41:46 GMT\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Content-Type\": [\"application/json\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc00089c1e0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: \"...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:127]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0005ef0e0\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-f9d294/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-f9d294/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc001357950\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Envoy-Upstream-Service-Time\": [\"166\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"a8f26788-567b-4640-a811-43c8b7434031\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 2EC63483A147487995A6328397F09705 Ref B: CHI261140505054 Ref C: 2026-04-04T02:44:04Z\",\n                    ],\n                    \"Date\": [\n                        \"Sat, 04 Apr 2026 02:44:04 GMT\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260404T024405Z:a8f26788-567b-4640-a811-43c8b7434031\",\n                    ],\n                    \"Content-Type\": [\"application/json\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/52040d45-044c-4511-88b7-9733c92944b6\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Request-Id\": [\n                        \"12919637-123e-4c90-833e-437777c2019f\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0005eefa0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:127]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0005ef0e0\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-f9d294/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-f9d294/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc001357950\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Envoy-Upstream-Service-Time\": [\"166\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"a8f26788-567b-4640-a811-43c8b7434031\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 2EC63483A147487995A6328397F09705 Ref B: CHI261140505054 Ref C: 2026-04-04T02:44:04Z\",\n                    ],\n                    \"Date\": [\n                        \"Sat, 04 Apr 2026 02:44:04 GMT\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260404T024405Z:a8f26788-567b-4640-a811-43c8b7434031\",\n                    ],\n                    \"Content-Type\": [\"application/json\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/52040d45-044c-4511-88b7-9733c92944b6\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Request-Id\": [\n                        \"12919637-123e-4c90-833e-437777c2019f\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0005eefa0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                        Host: ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:164]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000786700\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-tcr7xr-w9cl9v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-tcr7xr/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-tcr7xr-w9cl9v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-tcr7xr/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc000984f90\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260403T024529Z:507906a9-3741-4c35-99cf-b4f6b8fe9a4b\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 4BC4BC7D598E4A38B774049176C0C5B8 Ref B: CHI261140508025 Ref C: 2026-04-03T02:45:28Z\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/75948d99-9fdd-49dc-a774-686a518c88f9\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Date\": [\n                        \"Fri, 03 Apr 2026 02:45:28 GMT\",\n                    ],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Ms-Request-Id\": [\n                        \"e9eebcd1-0444-496b-90c5-43c9c0a830e0\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"247\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"507906a9-3741-4c35-99cf-b4f6b8fe9a4b\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Type\": [\"application/json\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc000786640\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n        ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:164]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000786700\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-tcr7xr-w9cl9v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-tcr7xr/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-tcr7xr-w9cl9v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-tcr7xr/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc000984f90\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260403T024529Z:507906a9-3741-4c35-99cf-b4f6b8fe9a4b\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 4BC4BC7D598E4A38B774049176C0C5B8 Ref B: CHI261140508025 Ref C: 2026-04-03T02:45:28Z\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/75948d99-9fdd-49dc-a774-686a518c88f9\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Date\": [\n                        \"Fri, 03 Apr 2026 02:45:28 GMT\",\n                    ],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Ms-Request-Id\": [\n                        \"e9eebcd1-0444-496b-90c5-43c9c0a830e0\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"247\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"507906a9-3741-4c35-99cf-b4f6b8fe9a4b\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Type\": [\"application/json\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc000786640\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n        ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040610572591960064","occurred_at":"2026-04-05T02:00:47Z","signature_id":"cfdca4180f7bdc0f29c75b9bd2b24a00c3a8792b4e4bcfd205042cb366cd570c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040248180829851648","occurred_at":"2026-04-04T02:00:46Z","signature_id":"d7688ab63a389ea09f8ad41fad1bbee654d86cd103c78d4ba4346ea44fd4f632","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039885780997902336","occurred_at":"2026-04-03T02:00:44Z","signature_id":"12d7954d434cf729c3b529273bdcd87aaa6a83e128da23b6acbf62fb7de8121a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039885780997902336","occurred_at":"2026-04-03T02:00:44Z","signature_id":"963c0e1ca6dca005aca4677bd88b36175b4895ea069525430fddb4fbb4910bcc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039885780997902336","occurred_at":"2026-04-03T02:00:44Z","signature_id":"d8d7691f6d3d966719306aa105447f9e144b72c8672c3385e1c0ab9bc18e0c23","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","signature_id":"4d26299f52b0b8986f5cb45926b1e2da9613d8a375ee0a382f06b7e2c7a01945","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","signature_id":"6eacec424ad1161d7aacbab03f8a93ea1c58263a875329c1260729344930c215","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","signature_id":"7a56671f4aff56ecc03b701290aceff3662c37535980cb96bf3f4b76002fa5d3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","signature_id":"a065c8647285da4ae861666481d317b07db2efdc79edc92e410f5b8bacb3f1ae","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","signature_id":"d3286a20e487ac27f27dee83ac29804121c9b49a8c8b8ea4d003f723f9be0d69","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","signature_id":"efed37f596e7501dc50ed6fc82727bd5d2db1dd41b58a065c7155228aa4d0495","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dds6fm, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-dds6fm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-dds6fm, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":10,"runs_affected":5,"failed_runs":5,"run_impact_percent":25,"anchor_occurrences":10,"after_last_push_count":10,"trend_counts":[0,5,4,0,0,0,1,0],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-02","2026-03-16","2026-03-23"],"prior_runs_affected":15,"prior_last_seen_at":"2026-03-26T19:56:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013345a0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-hmmjkl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl, error: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-hmmjkl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0010337e0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl\",\n            },\n            \u003c*fmt.wrapErrors | 0xc001334570\u003e{\n                msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-hmmjkl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0010337e0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013345a0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-hmmjkl\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl, error: failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-hmmjkl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0010337e0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl\",\n            },\n            \u003c*fmt.wrapErrors | 0xc001334570\u003e{\n                msg: \"failed waiting for cluster=\\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-hmmjkl\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc0010337e0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-hmmjkl\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:135]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000916240\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-pgdlrr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-pgdlrr, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-pgdlrr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-pgdlrr, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0010acbc0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-pgdlrr\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:135]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000916240\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-pgdlrr\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-pgdlrr, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-pgdlrr\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-pgdlrr, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0010acbc0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-pgdlrr\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:136]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001750cf0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-qdzg54\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54, error: failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-qdzg54\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000fe0f60\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54\",\n            },\n            \u003c*fmt.wrapErrors | 0xc001750cc0\u003e{\n                msg: \"failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-qdzg54\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000fe0f60\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:136]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001750cf0\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-qdzg54\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54, error: failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-qdzg54\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000fe0f60\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54\",\n            },\n            \u003c*fmt.wrapErrors | 0xc001750cc0\u003e{\n                msg: \"failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-qdzg54\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000fe0f60\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-qdzg54\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040248180829851648","occurred_at":"2026-04-04T02:00:46Z","signature_id":"c9bea9c76d2740349bda369a455dda33830aa97646af9e628ce09a94bc8774fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038973601599721472","occurred_at":"2026-03-31T13:36:01Z","signature_id":"9382cb9eb79eae42cc5dace1ccccc98e82e38345c45f04052607b8bfad63da73","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038798596739960832","occurred_at":"2026-03-31T02:00:39Z","signature_id":"347defe85f9f6eb0c1f25c4ce72b2ff46edca527ac9746d258764abc2fb5baf2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038798596739960832","occurred_at":"2026-03-31T02:00:39Z","signature_id":"7d98ec92bd51c96dc08930d9f98e3246a9a1e48679cd5d21d4b53730fe1fa926","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038798596739960832","occurred_at":"2026-03-31T02:00:39Z","signature_id":"c98f63922f49eccb84d159af311d73ccaa7e94b2fcfdca094ca21546834d5777","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038765373188214784","occurred_at":"2026-03-30T23:48:38Z","signature_id":"450fc46eb897ae03418c2fe9eac2322e9500a3838e192b479c4a2a2393594121","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038765373188214784","occurred_at":"2026-03-30T23:48:38Z","signature_id":"cfdc42f7f5e1bb71834c753a6705995745ffdeec1d0a9fde66b9179df2367e2d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038721439078551552","occurred_at":"2026-03-30T20:54:03Z","signature_id":"1a813e68e6ddf8f8546a168c340941d04d5b6ea6a1c70f99a114e33c5613f7df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038721439078551552","occurred_at":"2026-03-30T20:54:03Z","signature_id":"dde9a81a90c6df6068b9056997e34d925cd5b8143c637373a77e989a1b24dd84","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038721439078551552","occurred_at":"2026-03-30T20:54:03Z","signature_id":"fbefb27382b80d49faff96db2dd21ef9d35a0bc5238a6eeb1600aac3e05ac1c9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"9d240f3596a0d7016c609b7924c0509e3c0ca3a68bbc6af54e7ce19b76a8b991","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":10,"runs_affected":2,"failed_runs":2,"run_impact_percent":10,"anchor_occurrences":10,"after_last_push_count":10,"trend_counts":[0,0,0,0,10,0,0,0],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000732640\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-7mzwzp: failed waiting for cluster=\"cluster-zstream-4-20-7mzwzp\" in resourcegroup=\"rg-zstream-upgrade-4-20-tq69zx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\",\n      \"name\": \"e3ee4618-26eb-473e-867f-a23c3644e2bb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-04-02T17:31:57.472030714Z\",\n      \"endTime\": \"2026-04-02T17:32:55.049606818Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-20-7mzwzp: failed waiting for cluster=\\\"cluster-zstream-4-20-7mzwzp\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tq69zx\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\\\",\\n  \\\"name\\\": \\\"e3ee4618-26eb-473e-867f-a23c3644e2bb\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-04-02T17:31:57.472030714Z\\\",\\n  \\\"endTime\\\": \\\"2026-04-02T17:32:55.049606818Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000732600\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-7mzwzp\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tq69zx\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\\\",\\n  \\\"name\\\": \\\"e3ee4618-26eb-473e-867f-a23c3644e2bb\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-04-02T17:31:57.472030714Z\\\",\\n  \\\"endTime\\\": \\\"2026-04-02T17:32:55.049606818Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0014af200\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadaeast/81439e1d-c310-4abd-91e5-f37963acb072\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"499\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 41853D22B093487EB33229F443FF6553 Ref B: CHI261140513025 Ref C: 2026-04-02T17:33:00Z\",\n                   ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000732640\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-7mzwzp: failed waiting for cluster=\"cluster-zstream-4-20-7mzwzp\" in resourcegroup=\"rg-zstream-upgrade-4-20-tq69zx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\",\n      \"name\": \"e3ee4618-26eb-473e-867f-a23c3644e2bb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-04-02T17:31:57.472030714Z\",\n      \"endTime\": \"2026-04-02T17:32:55.049606818Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-20-7mzwzp: failed waiting for cluster=\\\"cluster-zstream-4-20-7mzwzp\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tq69zx\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\\\",\\n  \\\"name\\\": \\\"e3ee4618-26eb-473e-867f-a23c3644e2bb\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-04-02T17:31:57.472030714Z\\\",\\n  \\\"endTime\\\": \\\"2026-04-02T17:32:55.049606818Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000732600\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-7mzwzp\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tq69zx\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e3ee4618-26eb-473e-867f-a23c3644e2bb\\\",\\n  \\\"name\\\": \\\"e3ee4618-26eb-473e-867f-a23c3644e2bb\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-04-02T17:31:57.472030714Z\\\",\\n  \\\"endTime\\\": \\\"2026-04-02T17:32:55.049606818Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0014af200\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadaeast/81439e1d-c310-4abd-91e5-f37963acb072\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"499\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 41853D22B093487EB33229F443FF6553 Ref B: CHI261140513025 Ref C: 2026-04-02T17:33:00Z\",\n                   ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0008d7580\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-456qg5: failed waiting for cluster=\"cluster-zstream-4-19-456qg5\" in resourcegroup=\"rg-zstream-upgrade-4-19-p762zb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\",\n      \"name\": \"78ee839b-27e8-45ab-9bef-ab361680e06a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-04-02T17:27:46.835065402Z\",\n      \"endTime\": \"2026-04-02T17:27:57.932415489Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-19-456qg5: failed waiting for cluster=\\\"cluster-zstream-4-19-456qg5\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-p762zb\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\\\",\\n  \\\"name\\\": \\\"78ee839b-27e8-45ab-9bef-ab361680e06a\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-04-02T17:27:46.835065402Z\\\",\\n  \\\"endTime\\\": \\\"2026-04-02T17:27:57.932415489Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0008d7560\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-19-456qg5\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-p762zb\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\\\",\\n  \\\"name\\\": \\\"78ee839b-27e8-45ab-9bef-ab361680e06a\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-04-02T17:27:46.835065402Z\\\",\\n  \\\"endTime\\\": \\\"2026-04-02T17:27:57.932415489Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc001317380\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Length\": [\"499\"],\n                        \"X-Ms-Request-Id\": [\n                            \"f6c45648-41ff-430b-8947-420c54510b28\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260402T172807Z:e6630a99-063f-431a-9926-65553b1b3548\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/26acb1f6-ebb4-4c9f-9e33-8c891414cd26\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: C88F82DD8BF54F1BA57C9FED21B63FDF Ref B: CHI261140514034 Ref C: 2026-04-02T17:28:07Z\",\n                        ],\n                        \"Date\": [\n                            \"Thu, 02 Apr 2026 17:28:07 GMT\",\n                        ],\n...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0008d7580\u003e: \n    failed to create HCP cluster cluster-zstream-4-19-456qg5: failed waiting for cluster=\"cluster-zstream-4-19-456qg5\" in resourcegroup=\"rg-zstream-upgrade-4-19-p762zb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\",\n      \"name\": \"78ee839b-27e8-45ab-9bef-ab361680e06a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-04-02T17:27:46.835065402Z\",\n      \"endTime\": \"2026-04-02T17:27:57.932415489Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster cluster-zstream-4-19-456qg5: failed waiting for cluster=\\\"cluster-zstream-4-19-456qg5\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-p762zb\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\\\",\\n  \\\"name\\\": \\\"78ee839b-27e8-45ab-9bef-ab361680e06a\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-04-02T17:27:46.835065402Z\\\",\\n  \\\"endTime\\\": \\\"2026-04-02T17:27:57.932415489Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0008d7560\u003e{\n            msg: \"failed waiting for cluster=\\\"cluster-zstream-4-19-456qg5\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-p762zb\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78ee839b-27e8-45ab-9bef-ab361680e06a\\\",\\n  \\\"name\\\": \\\"78ee839b-27e8-45ab-9bef-ab361680e06a\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-04-02T17:27:46.835065402Z\\\",\\n  \\\"endTime\\\": \\\"2026-04-02T17:27:57.932415489Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc001317380\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Pragma\": [\"no-cache\"],\n                        \"Content-Length\": [\"499\"],\n                        \"X-Ms-Request-Id\": [\n                            \"f6c45648-41ff-430b-8947-420c54510b28\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260402T172807Z:e6630a99-063f-431a-9926-65553b1b3548\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/26acb1f6-ebb4-4c9f-9e33-8c891414cd26\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: C88F82DD8BF54F1BA57C9FED21B63FDF Ref B: CHI261140514034 Ref C: 2026-04-02T17:28:07Z\",\n                        ],\n                        \"Date\": [\n                            \"Thu, 02 Apr 2026 17:28:07 GMT\",\n                        ],\n...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0005873e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-qhj6bm: failed waiting for cluster=\"np-version-upgrade-cluster-qhj6bm\" in resourcegroup=\"rg-np-version-upgrade-qhj6bm-hp2rjl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\",\n      \"name\": \"774a88cb-0063-4a09-b407-ee6a7f59a9e6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-04-02T17:22:52.081012471Z\",\n      \"endTime\": \"2026-04-02T17:23:13.48665448Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster np-version-upgrade-cluster-qhj6bm: failed waiting for cluster=\\\"np-version-upgrade-cluster-qhj6bm\\\" in resourcegroup=\\\"rg-np-version-upgrade-qhj6bm-hp2rjl\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\\\",\\n  \\\"name\\\": \\\"774a88cb-0063-4a09-b407-ee6a7f59a9e6\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-04-02T17:22:52.081012471Z\\\",\\n  \\\"endTime\\\": \\\"2026-04-02T17:23:13.48665448Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0005873c0\u003e{\n            msg: \"failed waiting for cluster=\\\"np-version-upgrade-cluster-qhj6bm\\\" in resourcegroup=\\\"rg-np-version-upgrade-qhj6bm-hp2rjl\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\\\",\\n  \\\"name\\\": \\\"774a88cb-0063-4a09-b407-ee6a7f59a9e6\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-04-02T17:22:52.081012471Z\\\",\\n  \\\"endTime\\\": \\\"2026-04-02T17:23:13.48665448Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0010284e0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"498\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"8a74d4be-8a1d-4808-936e-22bae5b4ccd3\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 37CA783E55BC4DE4BA0F59B834B5F490 Ref B: CHI261140504062 Ref C: 2026-04-02T17:23:22Z\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"20\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260402T172322Z:8a74d4be-8a1d-4808-936e-22bae5b4ccd3\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Request-Id\": [\n                            \"c674f25c-3bae-4b96-9f1e...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0005873e0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-qhj6bm: failed waiting for cluster=\"np-version-upgrade-cluster-qhj6bm\" in resourcegroup=\"rg-np-version-upgrade-qhj6bm-hp2rjl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\",\n      \"name\": \"774a88cb-0063-4a09-b407-ee6a7f59a9e6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-04-02T17:22:52.081012471Z\",\n      \"endTime\": \"2026-04-02T17:23:13.48665448Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"insufficient public IP address quota: required 2, available 0\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster np-version-upgrade-cluster-qhj6bm: failed waiting for cluster=\\\"np-version-upgrade-cluster-qhj6bm\\\" in resourcegroup=\\\"rg-np-version-upgrade-qhj6bm-hp2rjl\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\\\",\\n  \\\"name\\\": \\\"774a88cb-0063-4a09-b407-ee6a7f59a9e6\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-04-02T17:22:52.081012471Z\\\",\\n  \\\"endTime\\\": \\\"2026-04-02T17:23:13.48665448Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0005873c0\u003e{\n            msg: \"failed waiting for cluster=\\\"np-version-upgrade-cluster-qhj6bm\\\" in resourcegroup=\\\"rg-np-version-upgrade-qhj6bm-hp2rjl\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/774a88cb-0063-4a09-b407-ee6a7f59a9e6\\\",\\n  \\\"name\\\": \\\"774a88cb-0063-4a09-b407-ee6a7f59a9e6\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-04-02T17:22:52.081012471Z\\\",\\n  \\\"endTime\\\": \\\"2026-04-02T17:23:13.48665448Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0010284e0\u003e{\n                ErrorCode: \"InternalServerError\",\n                StatusCode: 200,\n                RawResponse: {\n                    Status: \"200 OK\",\n                    StatusCode: 200,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"498\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"8a74d4be-8a1d-4808-936e-22bae5b4ccd3\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 37CA783E55BC4DE4BA0F59B834B5F490 Ref B: CHI261140504062 Ref C: 2026-04-02T17:23:22Z\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"20\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"CANADACENTRAL:20260402T172322Z:8a74d4be-8a1d-4808-936e-22bae5b4ccd3\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Request-Id\": [\n                            \"c674f25c-3bae-4b96-9f1e...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039753351280726016","occurred_at":"2026-04-02T17:14:28Z","signature_id":"20e46711d3e35710a980843025c80ad4a7b4cde8409fdf3d9c4a64498fb53b05","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039753351280726016","occurred_at":"2026-04-02T17:14:28Z","signature_id":"355229726f681e41eedcc021a5f6f4da31e67f116c3a8985a272f805822bcf87","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039753351280726016","occurred_at":"2026-04-02T17:14:28Z","signature_id":"42f9dbe161009cc54c6fd91be6ef6725fe5af524a9c492cbb4f914f791198d53","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039753351280726016","occurred_at":"2026-04-02T17:14:28Z","signature_id":"5555990c760129d0559f6b21d12a328fcac98cdae5cc03808dcd0781d1280bf1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039753351280726016","occurred_at":"2026-04-02T17:14:28Z","signature_id":"9b65bed4ba440d31a2069975341012840e9a50435e2711db9b3bacbab843329f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","occurred_at":"2026-04-02T08:48:43Z","signature_id":"69b22aaa698b3b1760bc99ff5999083f5c0c44c4dcf517eb76bf27f757a7e495","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","occurred_at":"2026-04-02T08:48:43Z","signature_id":"6e970d09796e67482e58fb3616052cfbda9bac0d41b02e47fa38c4737f014b34","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","occurred_at":"2026-04-02T08:48:43Z","signature_id":"9a9415a468f807e3f900a81033319d4a094d50d5a9a6a438265cad1fac141e42","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","occurred_at":"2026-04-02T08:48:43Z","signature_id":"e12a845dc4147e276bce346cf5007864692316e60bb487e5d88e3368003a798a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","occurred_at":"2026-04-02T08:48:43Z","signature_id":"e3fcff74c9339554323e999ccbc70bc28db346f3dda20bfd1db933da9e84e721","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"e17b72ff49490f6e1940fa5deb5057f7012ebd4b2cee2f2d2e8b8d47e6784bb9","failure_pattern":"ERROR CODE: LockContention; detail message Failed to acquire lock: timed out; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LockContention","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":7,"runs_affected":2,"failed_runs":2,"run_impact_percent":10,"anchor_occurrences":7,"after_last_push_count":11,"trend_counts":[0,5,0,0,2,0,0,0],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-02","2026-03-09"],"prior_runs_affected":6,"prior_last_seen_at":"2026-03-13T06:56:30Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009f2840\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-k2qhj7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: LockContention\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LockContention\",\n        \"message\": \"Failed to acquire lock: timed out\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-k2qhj7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"LockContention\\\",\\n    \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc00159e990\u003e{\n            ErrorCode: \"LockContention\",\n            StatusCode: 503,\n            RawResponse: {\n                Status: \"503 Service Unavailable\",\n                StatusCode: 503,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/1c5d69eb-b218-43c3-89e1-797546221c53\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Date\": [\n                        \"Thu, 02 Apr 2026 02:10:08 GMT\",\n                    ],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Error-Code\": [\"LockContention\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260402T021008Z:afdf4447-88c0-4010-85eb-2250eadd30fe\",\n                    ],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 1EC287F8926644E099B85CCB3B5E9590 Ref B: CHI261140504042 Ref C: 2026-04-02T02:09:58Z\",\n                    ],\n                    \"Content-Length\": [\"186\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Request-Id\": [\n                        \"2b953333-0bff-468f-ad05-d66baee5ebf5\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"10006\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"afdf4447-88c0-4010-85eb-2250eadd30fe\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Retry-After\": [\"10\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0009f2700\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 76, 111, 99, 107, 67, 111, 110, 116, 101, 110, 116, 105, 111, 110, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 32, 116, 111, 32, 97, 99, 113, 117, 105, 114, 101, 32, 108, 111, 99, 107, 58, 32, 116, 105, 109, 101, 100, 32, 111, 117, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 57, 51, 57, 57, 50, 56, 49, 45, 48, 48, 97, 50, 45, 52, 98, 51, 57, 45, 98, 98, 51, 100, 45, 98, 50, 54, 52, 53, 98, 98, 98, 100, 98, 57, 51, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                Conte...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009f2840\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-k2qhj7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: LockContention\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LockContention\",\n        \"message\": \"Failed to acquire lock: timed out\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-k2qhj7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"LockContention\\\",\\n    \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc00159e990\u003e{\n            ErrorCode: \"LockContention\",\n            StatusCode: 503,\n            RawResponse: {\n                Status: \"503 Service Unavailable\",\n                StatusCode: 503,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/1c5d69eb-b218-43c3-89e1-797546221c53\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Date\": [\n                        \"Thu, 02 Apr 2026 02:10:08 GMT\",\n                    ],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Error-Code\": [\"LockContention\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260402T021008Z:afdf4447-88c0-4010-85eb-2250eadd30fe\",\n                    ],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 1EC287F8926644E099B85CCB3B5E9590 Ref B: CHI261140504042 Ref C: 2026-04-02T02:09:58Z\",\n                    ],\n                    \"Content-Length\": [\"186\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Request-Id\": [\n                        \"2b953333-0bff-468f-ad05-d66baee5ebf5\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"10006\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"afdf4447-88c0-4010-85eb-2250eadd30fe\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Retry-After\": [\"10\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0009f2700\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 76, 111, 99, 107, 67, 111, 110, 116, 101, 110, 116, 105, 111, 110, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 32, 116, 111, 32, 97, 99, 113, 117, 105, 114, 101, 32, 108, 111, 99, 107, 58, 32, 116, 105, 109, 101, 100, 32, 111, 117, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 57, 51, 57, 57, 50, 56, 49, 45, 48, 48, 97, 50, 45, 52, 98, 51, 57, 45, 98, 98, 51, 100, 45, 98, 50, 54, 52, 53, 98, 98, 98, 100, 98, 57, 51, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                Conte...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:103]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00142ee20\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-4qqv6m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: LockContention\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LockContention\",\n        \"message\": \"Failed to acquire lock: timed out\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-4qqv6m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"LockContention\\\",\\n    \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc00104d530\u003e{\n            ErrorCode: \"LockContention\",\n            StatusCode: 503,\n            RawResponse: {\n                Status: \"503 Service Unavailable\",\n                StatusCode: 503,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/0949565f-2116-4c2d-ac42-f874e16826f1\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260402T021009Z:c60ff00c-0498-471e-83f9-643c3970b3df\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 84A2B7061A1D427CAC14B9844929E76E Ref B: CHI261140502054 Ref C: 2026-04-02T02:09:58Z\",\n                    ],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Length\": [\"186\"],\n                    \"Retry-After\": [\"10\"],\n                    \"X-Ms-Error-Code\": [\"LockContention\"],\n                    \"X-Ms-Request-Id\": [\n                        \"342b364f-60a1-493c-a980-5f75e6f1c77f\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"10010\"],\n                    \"Date\": [\n                        \"Thu, 02 Apr 2026 02:10:08 GMT\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"c60ff00c-0498-471e-83f9-643c3970b3df\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc00142ed60\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 76, 111, 99, 107, 67, 111, 110, 116, 101, 110, 116, 105, 111, 110, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 32, 116, 111, 32, 97, 99, 113, 117, 105, 114, 101, 32, 108, 111, 99, 107, 58, 32, 116, 105, 109, 101, 100, 32, 111, 117, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 57, 51, 57, 57, 50, 56, 49, 45, 48, 48, 97, 50, 45, 52, 98, 51, 57, 45, 98, 98, 51, 100, 45, 98, 50, 54, 52, 53, 98, 98, 98, 100, 98, 57, 51, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                Co...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:103]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00142ee20\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-4qqv6m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: LockContention\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LockContention\",\n        \"message\": \"Failed to acquire lock: timed out\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-4qqv6m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"LockContention\\\",\\n    \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc00104d530\u003e{\n            ErrorCode: \"LockContention\",\n            StatusCode: 503,\n            RawResponse: {\n                Status: \"503 Service Unavailable\",\n                StatusCode: 503,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/0949565f-2116-4c2d-ac42-f874e16826f1\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260402T021009Z:c60ff00c-0498-471e-83f9-643c3970b3df\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 84A2B7061A1D427CAC14B9844929E76E Ref B: CHI261140502054 Ref C: 2026-04-02T02:09:58Z\",\n                    ],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Length\": [\"186\"],\n                    \"Retry-After\": [\"10\"],\n                    \"X-Ms-Error-Code\": [\"LockContention\"],\n                    \"X-Ms-Request-Id\": [\n                        \"342b364f-60a1-493c-a980-5f75e6f1c77f\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"10010\"],\n                    \"Date\": [\n                        \"Thu, 02 Apr 2026 02:10:08 GMT\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"c60ff00c-0498-471e-83f9-643c3970b3df\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc00142ed60\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 76, 111, 99, 107, 67, 111, 110, 116, 101, 110, 116, 105, 111, 110, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 32, 116, 111, 32, 97, 99, 113, 117, 105, 114, 101, 32, 108, 111, 99, 107, 58, 32, 116, 105, 109, 101, 100, 32, 111, 117, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 57, 51, 57, 57, 50, 56, 49, 45, 48, 48, 97, 50, 45, 52, 98, 51, 57, 45, 98, 98, 51, 100, 45, 98, 50, 54, 52, 53, 98, 98, 98, 100, 98, 57, 51, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                Co...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000e760c0\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-pkkxdw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: LockContention\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LockContention\",\n        \"message\": \"Failed to acquire lock: timed out\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-pkkxdw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"LockContention\\\",\\n    \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc000dce1e0\u003e{\n            ErrorCode: \"LockContention\",\n            StatusCode: 503,\n            RawResponse: {\n                Status: \"503 Service Unavailable\",\n                StatusCode: 503,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"Content-Length\": [\"186\"],\n                    \"X-Ms-Error-Code\": [\"LockContention\"],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Retry-After\": [\"10\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"37ef49af-09e9-40ee-bfca-efc082a80f64\",\n                    ],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260330T021725Z:37ef49af-09e9-40ee-bfca-efc082a80f64\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Request-Id\": [\n                        \"e0a372c3-fcdc-4ce9-bf08-e2fce9bc2797\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"10011\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/3345b2b1-e23f-4ce2-8dab-587f9642a08d\",\n                    ],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 93A8B2C021534273A8B574371656B5C5 Ref B: CH1AA2020620025 Ref C: 2026-03-30T02:17:14Z\",\n                    ],\n                    \"Date\": [\n                        \"Mon, 30 Mar 2026 02:17:24 GMT\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc000e76000\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 76, 111, 99, 107, 67, 111, 110, 116, 101, 110, 116, 105, 111, 110, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 32, 116, 111, 32, 97, 99, 113, 117, 105, 114, 101, 32, 108, 111, 99, 107, 58, 32, 116, 105, 109, 101, 100, 32, 111, 117, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 57, 51, 57, 57, 50, 56, 49, 45, 48, 48, 97, 50, 45, 52, 98, 51, 57, 45, 98, 98, 51, 100, 45, 98, 50, 54, 52, 53, 98, 98, 98, 100, 98, 57, 51, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n          ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000e760c0\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-pkkxdw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: LockContention\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LockContention\",\n        \"message\": \"Failed to acquire lock: timed out\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-pkkxdw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-ver-4-19/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: LockContention\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"LockContention\\\",\\n    \\\"message\\\": \\\"Failed to acquire lock: timed out\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc000dce1e0\u003e{\n            ErrorCode: \"LockContention\",\n            StatusCode: 503,\n            RawResponse: {\n                Status: \"503 Service Unavailable\",\n                StatusCode: 503,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"Content-Length\": [\"186\"],\n                    \"X-Ms-Error-Code\": [\"LockContention\"],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Retry-After\": [\"10\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"37ef49af-09e9-40ee-bfca-efc082a80f64\",\n                    ],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260330T021725Z:37ef49af-09e9-40ee-bfca-efc082a80f64\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Request-Id\": [\n                        \"e0a372c3-fcdc-4ce9-bf08-e2fce9bc2797\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"10011\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/3345b2b1-e23f-4ce2-8dab-587f9642a08d\",\n                    ],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: 93A8B2C021534273A8B574371656B5C5 Ref B: CH1AA2020620025 Ref C: 2026-03-30T02:17:14Z\",\n                    ],\n                    \"Date\": [\n                        \"Mon, 30 Mar 2026 02:17:24 GMT\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc000e76000\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 76, 111, 99, 107, 67, 111, 110, 116, 101, 110, 116, 105, 111, 110, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 70, 97, 105, 108, 101, 100, 32, 116, 111, 32, 97, 99, 113, 117, 105, 114, 101, 32, 108, 111, 99, 107, 58, 32, 116, 105, 109, 101, 100, 32, 111, 117, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 57, 57, 51, 57, 57, 50, 56, 49, 45, 48, 48, 97, 50, 45, 52, 98, 51, 57, 45, 98, 98, 51, 100, 45, 98, 50, 54, 52, 53, 98, 98, 98, 100, 98, 57, 51, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n          ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","signature_id":"61fec9e174cbe2103641b0ca2d00e068a13dee8925feb67a852f2d37175153ed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039517455973355520","occurred_at":"2026-04-02T01:37:08Z","signature_id":"ad02bdcbba8a2cd55fe28294470d643c39c011c3cd16fba715a90713c1dd2990","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038436115614011392","occurred_at":"2026-03-30T02:00:16Z","signature_id":"481c4f5b5c4b9602a3adf1de8af50eb014ef0d1b21513a48a1df5be1e1d57903","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038436115614011392","occurred_at":"2026-03-30T02:00:16Z","signature_id":"592717ee6fed046c085b3e31f946cd65b6c891dfa12ffbcbaf7a9a65cfecaca9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038436115614011392","occurred_at":"2026-03-30T02:00:16Z","signature_id":"81a2e7894d0a2d52c2fdba0a657e1d3c5f15808605560f628ee0ed550bc32910","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038436115614011392","occurred_at":"2026-03-30T02:00:16Z","signature_id":"8a563dbb5aa2c07f1dd5f4f8dbe4169727c979c71a1a480f519a8c623e29f518","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038436115614011392","occurred_at":"2026-03-30T02:00:16Z","signature_id":"ab1af64cd2fd0073bcb42ad4fcc3577c446d580b5876bb1adc8b20453ca9a690","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"47c4fe17c8b40b438461015820e355e731ca2f90428cea3ac600911bb9a03df7","failure_pattern":"ERROR CODE: ServiceUnavailable; detail message Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ServiceUnavailable","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":5,"anchor_occurrences":3,"after_last_push_count":11,"trend_counts":[0,0,0,0,3,0,0,0],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-02","2026-03-23"],"prior_runs_affected":4,"prior_last_seen_at":"2026-03-25T19:15:18Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00075c1e0\u003e: \n    failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-wvg7xj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-wvg7xj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: ServiceUnavailable\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ServiceUnavailable\",\n        \"message\": \"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster patch-name-cluster: failed starting cluster creation \\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-wvg7xj\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-wvg7xj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ServiceUnavailable\\\",\\n    \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00075c1c0\u003e{\n            msg: \"failed starting cluster creation \\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-wvg7xj\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-wvg7xj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ServiceUnavailable\\\",\\n    \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00004d2f0\u003e{\n                ErrorCode: \"ServiceUnavailable\",\n                StatusCode: 503,\n                RawResponse: {\n                    Status: \"503 Service Unavailable\",\n                    StatusCode: 503,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"2878\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"0b9741c0-9777-4781-b4fb-bf1afce41505\",\n                        ],\n                        \"Retry-After\": [\"59\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                        \"Date\": [\n                            \"Thu, 02 Apr 2026 08:57:47 GMT\",\n                        ],\n                        \"Content-Length\": [\"257\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Error-Code\": [\"ServiceUnavailable\"],\n                        \"X-Ms-Request-Id\": [\n                            \"87531355-f28f-4c12-bf8b-c7f8a31db6ce\",\n                        ],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/7ad18b62-8874-4d58-81d6-01921cd42fc7\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"UKSOUTH:20260402T085747Z:0b9741c0-9777-4781-b4fb-bf1afce41505\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00075c1e0\u003e: \n    failed to create HCP cluster patch-name-cluster: failed starting cluster creation \"patch-name-cluster\" in resourcegroup=\"patch-name-wvg7xj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-wvg7xj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: ServiceUnavailable\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ServiceUnavailable\",\n        \"message\": \"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster patch-name-cluster: failed starting cluster creation \\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-wvg7xj\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-wvg7xj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ServiceUnavailable\\\",\\n    \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc00075c1c0\u003e{\n            msg: \"failed starting cluster creation \\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-wvg7xj\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-wvg7xj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ServiceUnavailable\\\",\\n    \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00004d2f0\u003e{\n                ErrorCode: \"ServiceUnavailable\",\n                StatusCode: 503,\n                RawResponse: {\n                    Status: \"503 Service Unavailable\",\n                    StatusCode: 503,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"2878\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"0b9741c0-9777-4781-b4fb-bf1afce41505\",\n                        ],\n                        \"Retry-After\": [\"59\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                        \"Date\": [\n                            \"Thu, 02 Apr 2026 08:57:47 GMT\",\n                        ],\n                        \"Content-Length\": [\"257\"],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Error-Code\": [\"ServiceUnavailable\"],\n                        \"X-Ms-Request-Id\": [\n                            \"87531355-f28f-4c12-bf8b-c7f8a31db6ce\",\n                        ],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/7ad18b62-8874-4d58-81d6-01921cd42fc7\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"UKSOUTH:20260402T085747Z:0b9741c0-9777-4781-b4fb-bf1afce41505\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:123]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0004e6700\u003e: \n    failed starting cluster creation \"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-p6dr85\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-p6dr85/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster2\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: ServiceUnavailable\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ServiceUnavailable\",\n        \"message\": \"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed starting cluster creation \\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-p6dr85\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-p6dr85/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster2\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ServiceUnavailable\\\",\\n    \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc001402630\u003e{\n            ErrorCode: \"ServiceUnavailable\",\n            StatusCode: 503,\n            RawResponse: {\n                Status: \"503 Service Unavailable\",\n                StatusCode: 503,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Request-Id\": [\n                        \"981e3b54-276e-45f0-a48f-b05f015c79dc\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"8461\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/d750dcff-9cd4-44ec-8a65-6f1b69e57a9d\",\n                    ],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"45bdda71-4b5b-4b9b-9fd7-4df364bf7808\",\n                    ],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260402T085811Z:45bdda71-4b5b-4b9b-9fd7-4df364bf7808\",\n                    ],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Length\": [\"257\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Error-Code\": [\"ServiceUnavailable\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: C36632182EBE4B8EB52F9106BF8C5BB2 Ref B: CHI261140502029 Ref C: 2026-04-02T08:58:02Z\",\n                    ],\n                    \"Retry-After\": [\"59\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Date\": [\n                        \"Thu, 02 Apr 2026 08:58:11 GMT\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0004e6480\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 83, 101, 114, 118, 105, 99, 101, 85, 110, 97, 118, 97, 105, 108, 97, 98, 108, 101, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 108, 117, 115, 116, 101, 114, 32, 105, 110, 115, 116, 97, 108, 108, 97, 116, 105, 111, 110, 32, 104, 97, 115, 32, 102, 97, 105, 108, 101, 100, 32, 100, 117, 101, 32, 116, 111, 32, 117, 110, 97, 118, 97, 105, 108, 97, 98, 108, 101, 32, 99, 97, 112, 97, 99, 105, 116, 121, 46, 32, 87, 104, 105, 108, 101, 32, 119, 101, 32, 97, 114, 101, 32, 98, 114, 105, 110, 103, 105, 110, 103, 32, 111, 110, 32, 97, 100, 100, 105, 116, 105, 111, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:123]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0004e6700\u003e: \n    failed starting cluster creation \"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-p6dr85\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-p6dr85/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster2\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: ServiceUnavailable\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ServiceUnavailable\",\n        \"message\": \"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed starting cluster creation \\\"basic-hcp-cluster2\\\" in resourcegroup=\\\"customer-rg-p6dr85\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-p6dr85/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster2\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ServiceUnavailable\\\",\\n    \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc001402630\u003e{\n            ErrorCode: \"ServiceUnavailable\",\n            StatusCode: 503,\n            RawResponse: {\n                Status: \"503 Service Unavailable\",\n                StatusCode: 503,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Request-Id\": [\n                        \"981e3b54-276e-45f0-a48f-b05f015c79dc\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"8461\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/d750dcff-9cd4-44ec-8a65-6f1b69e57a9d\",\n                    ],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"45bdda71-4b5b-4b9b-9fd7-4df364bf7808\",\n                    ],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260402T085811Z:45bdda71-4b5b-4b9b-9fd7-4df364bf7808\",\n                    ],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Length\": [\"257\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Error-Code\": [\"ServiceUnavailable\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: C36632182EBE4B8EB52F9106BF8C5BB2 Ref B: CHI261140502029 Ref C: 2026-04-02T08:58:02Z\",\n                    ],\n                    \"Retry-After\": [\"59\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Date\": [\n                        \"Thu, 02 Apr 2026 08:58:11 GMT\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0004e6480\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 83, 101, 114, 118, 105, 99, 101, 85, 110, 97, 118, 97, 105, 108, 97, 98, 108, 101, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 108, 117, 115, 116, 101, 114, 32, 105, 110, 115, 116, 97, 108, 108, 97, 116, 105, 111, 110, 32, 104, 97, 115, 32, 102, 97, 105, 108, 101, 100, 32, 100, 117, 101, 32, 116, 111, 32, 117, 110, 97, 118, 97, 105, 108, 97, 98, 108, 101, 32, 99, 97, 112, 97, 99, 105, 116, 121, 46, 32, 87, 104, 105, 108, 101, 32, 119, 101, 32, 97, 114, 101, 32, 98, 114, 105, 110, 103, 105, 110, 103, 32, 111, 110, 32, 97, 100, 100, 105, 116, 105, 111, ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000880260\u003e: \n    failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-6tff8z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-6tff8z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: ServiceUnavailable\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ServiceUnavailable\",\n        \"message\": \"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster basic-cluster: failed starting cluster creation \\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-6tff8z\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-6tff8z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ServiceUnavailable\\\",\\n    \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000880200\u003e{\n            msg: \"failed starting cluster creation \\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-6tff8z\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-6tff8z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ServiceUnavailable\\\",\\n    \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0012ca1b0\u003e{\n                ErrorCode: \"ServiceUnavailable\",\n                StatusCode: 503,\n                RawResponse: {\n                    Status: \"503 Service Unavailable\",\n                    StatusCode: 503,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Request-Id\": [\n                            \"2a87bdad-8340-4cda-a740-2d7cc278c733\",\n                        ],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"294bd1e5-7abf-4f05-a3c2-931fa8a72c4e\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"UKSOUTH:20260402T085750Z:294bd1e5-7abf-4f05-a3c2-931fa8a72c4e\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"Date\": [\n                            \"Thu, 02 Apr 2026 08:57:50 GMT\",\n                        ],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"2649\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/cc9e8570-fcc2-4ba9-9d49-08bb67825537\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Content-Length\": [\"257\"],\n                        \"Retry-After\": [\"59\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Ms-Fail...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:106]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000880260\u003e: \n    failed to create HCP cluster basic-cluster: failed starting cluster creation \"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-6tff8z\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-6tff8z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 503: 503 Service Unavailable\n    ERROR CODE: ServiceUnavailable\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ServiceUnavailable\",\n        \"message\": \"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster basic-cluster: failed starting cluster creation \\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-6tff8z\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-6tff8z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ServiceUnavailable\\\",\\n    \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc000880200\u003e{\n            msg: \"failed starting cluster creation \\\"basic-cluster\\\" in resourcegroup=\\\"rg-cluster-nsg-subnet-reuse-6tff8z\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-6tff8z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 503: 503 Service Unavailable\\nERROR CODE: ServiceUnavailable\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"ServiceUnavailable\\\",\\n    \\\"message\\\": \\\"Cluster installation has failed due to unavailable capacity. While we are bringing on additional capacity, try your cluster installation in another region or try again later\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc0012ca1b0\u003e{\n                ErrorCode: \"ServiceUnavailable\",\n                StatusCode: 503,\n                RawResponse: {\n                    Status: \"503 Service Unavailable\",\n                    StatusCode: 503,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Ms-Request-Id\": [\n                            \"2a87bdad-8340-4cda-a740-2d7cc278c733\",\n                        ],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"294bd1e5-7abf-4f05-a3c2-931fa8a72c4e\",\n                        ],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"UKSOUTH:20260402T085750Z:294bd1e5-7abf-4f05-a3c2-931fa8a72c4e\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"Date\": [\n                            \"Thu, 02 Apr 2026 08:57:50 GMT\",\n                        ],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Envoy-Upstream-Service-Time\": [\"2649\"],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/cc9e8570-fcc2-4ba9-9d49-08bb67825537\",\n                        ],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                        \"X-Content-Type-Options\": [\"nosniff\"],\n                        \"Content-Length\": [\"257\"],\n                        \"Retry-After\": [\"59\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Pragma\": [\"no-cache\"],\n                        \"X-Ms-Fail...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","occurred_at":"2026-04-02T08:48:43Z","signature_id":"3a2e25e9d1890483fd272049272851235e7e4ec464629137b0a1a3d7b8980e7f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","occurred_at":"2026-04-02T08:48:43Z","signature_id":"cacb304bc7bd5ba62586475b5881017dd5ae0a6ac7b13d88e7aa4f249a8ed2ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039626069157351424","occurred_at":"2026-04-02T08:48:43Z","signature_id":"fd76359852bfe92722186a57e697c3c69f783a8157d4dcae452c9c7c11c50934","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"953e20e9d1db6d0f9c7e6c5ea18e02689f21a0690a145ecf8a28e443cb3a9c26","failure_pattern":"add password: error status code received from the API","search_query":"add password: error status code received from the API","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":10,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,1,1,0,0,0,0,0],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:252]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000517600\u003e: \n    add password: error status code received from the API\n    {\n        msg: \"add password: error status code received from the API\",\n        err: \u003c*odataerrors.ODataError | 0xc000b0ba40\u003e{\n            ApiError: {\n                Message: \"\",\n                ResponseStatusCode: 404,\n                ResponseHeaders: {\n                    header: {\n                        headers: {\n                            \"client-request-id\": {\n                                \"0c31e575-0500-4a44-aebc-155ee4c1eea2\": {},\n                            },\n                            \"content-type\": {\n                                \"application/json\": {},\n                            },\n                            \"x-ms-resource-unit\": {\"1\": {}},\n                            \"date\": {\n                                \"Tue, 31 Mar 2026 02:30:32 GMT\": {},\n                            },\n                            \"strict-transport-security\": {\n                                \"max-age=31536000\": {},\n                            },\n                            \"request-id\": {\n                                \"0c31e575-0500-4a44-aebc-155ee4c1eea2\": {},\n                            },\n                            \"x-ms-ags-diagnostic\": {\n                                \"{\\\"ServerInfo\\\":{\\\"DataCenter\\\":\\\"Canada Central\\\",\\\"Slice\\\":\\\"E\\\",\\\"Ring\\\":\\\"3\\\",\\\"ScaleUnit\\\":\\\"001\\\",\\\"RoleInstance\\\":\\\"YT1PEPF00003E4B\\\"}}\": {},\n                            },\n                            \"cache-control\": {\"no-cache\": {}},\n                            \"vary\": {\n                                \"Accept-Encoding\": {},\n                            },\n                        },\n                    },\n                },\n            },\n            backingStore: \u003c*store.InMemoryBackingStore | 0xc000517540\u003e{\n                returnOnlyChangedValues: false,\n                initializationCompleted: false,\n                store: {\n                    \"errorEscaped\": \u003c*odataerrors.MainError | 0xc000741cd0\u003e{\n                        backingStore: \u003c*store.InMemoryBackingStore | 0xc000517580\u003e{\n                            returnOnlyChangedValues: false,\n                            initializationCompleted: false,\n                            store: {\n                                \"innerError\": \u003c*odataerrors.InnerError | 0xc000741d30\u003e{\n                                    backingStore: \u003c*store.InMemoryBackingStore | 0xc0005175c0\u003e{\n                                        returnOnlyChangedValues: false,\n                                        initializationCompleted: false,\n                                        store: {...: ..., ...: ...},\n                                        subscribers: {},\n                                        changedValues: {...: ..., ...: ...},\n                                    },\n                                },\n                                \"code\": \u003c*string | 0xc000741a20\u003e\"Request_ResourceNotFound\",\n                                \"message\": \u003c*string | 0xc000741a70\u003e\"Resource '9c86f191-fe0d-40fc-8883-79c3daf62dcd' does not exist or one of its queried reference-property objects are not present.\",\n                            },\n                            subscribers: {},\n                            changedValues: {\"innerError\": false, \"code\": false, \"message\": false},\n                        },\n                    },\n                },\n                subscribers: {},\n                changedValues: {\n                    \"errorEscaped\": false,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:252]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000517600\u003e: \n    add password: error status code received from the API\n    {\n        msg: \"add password: error status code received from the API\",\n        err: \u003c*odataerrors.ODataError | 0xc000b0ba40\u003e{\n            ApiError: {\n                Message: \"\",\n                ResponseStatusCode: 404,\n                ResponseHeaders: {\n                    header: {\n                        headers: {\n                            \"client-request-id\": {\n                                \"0c31e575-0500-4a44-aebc-155ee4c1eea2\": {},\n                            },\n                            \"content-type\": {\n                                \"application/json\": {},\n                            },\n                            \"x-ms-resource-unit\": {\"1\": {}},\n                            \"date\": {\n                                \"Tue, 31 Mar 2026 02:30:32 GMT\": {},\n                            },\n                            \"strict-transport-security\": {\n                                \"max-age=31536000\": {},\n                            },\n                            \"request-id\": {\n                                \"0c31e575-0500-4a44-aebc-155ee4c1eea2\": {},\n                            },\n                            \"x-ms-ags-diagnostic\": {\n                                \"{\\\"ServerInfo\\\":{\\\"DataCenter\\\":\\\"Canada Central\\\",\\\"Slice\\\":\\\"E\\\",\\\"Ring\\\":\\\"3\\\",\\\"ScaleUnit\\\":\\\"001\\\",\\\"RoleInstance\\\":\\\"YT1PEPF00003E4B\\\"}}\": {},\n                            },\n                            \"cache-control\": {\"no-cache\": {}},\n                            \"vary\": {\n                                \"Accept-Encoding\": {},\n                            },\n                        },\n                    },\n                },\n            },\n            backingStore: \u003c*store.InMemoryBackingStore | 0xc000517540\u003e{\n                returnOnlyChangedValues: false,\n                initializationCompleted: false,\n                store: {\n                    \"errorEscaped\": \u003c*odataerrors.MainError | 0xc000741cd0\u003e{\n                        backingStore: \u003c*store.InMemoryBackingStore | 0xc000517580\u003e{\n                            returnOnlyChangedValues: false,\n                            initializationCompleted: false,\n                            store: {\n                                \"innerError\": \u003c*odataerrors.InnerError | 0xc000741d30\u003e{\n                                    backingStore: \u003c*store.InMemoryBackingStore | 0xc0005175c0\u003e{\n                                        returnOnlyChangedValues: false,\n                                        initializationCompleted: false,\n                                        store: {...: ..., ...: ...},\n                                        subscribers: {},\n                                        changedValues: {...: ..., ...: ...},\n                                    },\n                                },\n                                \"code\": \u003c*string | 0xc000741a20\u003e\"Request_ResourceNotFound\",\n                                \"message\": \u003c*string | 0xc000741a70\u003e\"Resource '9c86f191-fe0d-40fc-8883-79c3daf62dcd' does not exist or one of its queried reference-property objects are not present.\",\n                            },\n                            subscribers: {},\n                            changedValues: {\"innerError\": false, \"code\": false, \"message\": false},\n                        },\n                    },\n                },\n                subscribers: {},\n                changedValues: {\n                    \"errorEscaped\": false,\n                },\n            },\n        },\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:252]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000e25420\u003e: \n    add password: error status code received from the API\n    {\n        msg: \"add password: error status code received from the API\",\n        err: \u003c*odataerrors.ODataError | 0xc00063e4e0\u003e{\n            ApiError: {\n                Message: \"\",\n                ResponseStatusCode: 404,\n                ResponseHeaders: {\n                    header: {\n                        headers: {\n                            \"x-ms-ags-diagnostic\": {\n                                \"{\\\"ServerInfo\\\":{\\\"DataCenter\\\":\\\"East US\\\",\\\"Slice\\\":\\\"E\\\",\\\"Ring\\\":\\\"5\\\",\\\"ScaleUnit\\\":\\\"007\\\",\\\"RoleInstance\\\":\\\"MN1PEPF00028332\\\"}}\": {},\n                            },\n                            \"x-ms-resource-unit\": {\"1\": {}},\n                            \"date\": {\n                                \"Tue, 31 Mar 2026 00:21:31 GMT\": {},\n                            },\n                            \"vary\": {\n                                \"Accept-Encoding\": {},\n                            },\n                            \"request-id\": {\n                                \"061ef776-ae17-49c6-bbac-b0d62d7d9704\": {},\n                            },\n                            \"cache-control\": {\"no-cache\": {}},\n                            \"content-type\": {\n                                \"application/json\": {},\n                            },\n                            \"client-request-id\": {\n                                \"061ef776-ae17-49c6-bbac-b0d62d7d9704\": {},\n                            },\n                            \"strict-transport-security\": {\n                                \"max-age=31536000\": {},\n                            },\n                        },\n                    },\n                },\n            },\n            backingStore: \u003c*store.InMemoryBackingStore | 0xc000e253c0\u003e{\n                returnOnlyChangedValues: false,\n                initializationCompleted: false,\n                store: {\n                    \"errorEscaped\": \u003c*odataerrors.MainError | 0xc000d26ce0\u003e{\n                        backingStore: \u003c*store.InMemoryBackingStore | 0xc000e253e0\u003e{\n                            returnOnlyChangedValues: false,\n                            initializationCompleted: false,\n                            store: {\n                                \"code\": \u003c*string | 0xc000d26b60\u003e\"Request_ResourceNotFound\",\n                                \"message\": \u003c*string | 0xc000d26bb0\u003e\"Resource '02691b9b-c5f7-45b0-9cff-7fc2c9805fcd' does not exist or one of its queried reference-property objects are not present.\",\n                                \"innerError\": \u003c*odataerrors.InnerError | 0xc000d26d40\u003e{\n                                    backingStore: \u003c*store.InMemoryBackingStore | 0xc000e25400\u003e{\n                                        returnOnlyChangedValues: false,\n                                        initializationCompleted: false,\n                                        store: {...: ..., ...: ...},\n                                        subscribers: {},\n                                        changedValues: {...: ..., ...: ...},\n                                    },\n                                },\n                            },\n                            subscribers: {},\n                            changedValues: {\"code\": false, \"message\": false, \"innerError\": false},\n                        },\n                    },\n                },\n                subscribers: {},\n                changedValues: {\n                    \"errorEscaped\": false,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:252]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000e25420\u003e: \n    add password: error status code received from the API\n    {\n        msg: \"add password: error status code received from the API\",\n        err: \u003c*odataerrors.ODataError | 0xc00063e4e0\u003e{\n            ApiError: {\n                Message: \"\",\n                ResponseStatusCode: 404,\n                ResponseHeaders: {\n                    header: {\n                        headers: {\n                            \"x-ms-ags-diagnostic\": {\n                                \"{\\\"ServerInfo\\\":{\\\"DataCenter\\\":\\\"East US\\\",\\\"Slice\\\":\\\"E\\\",\\\"Ring\\\":\\\"5\\\",\\\"ScaleUnit\\\":\\\"007\\\",\\\"RoleInstance\\\":\\\"MN1PEPF00028332\\\"}}\": {},\n                            },\n                            \"x-ms-resource-unit\": {\"1\": {}},\n                            \"date\": {\n                                \"Tue, 31 Mar 2026 00:21:31 GMT\": {},\n                            },\n                            \"vary\": {\n                                \"Accept-Encoding\": {},\n                            },\n                            \"request-id\": {\n                                \"061ef776-ae17-49c6-bbac-b0d62d7d9704\": {},\n                            },\n                            \"cache-control\": {\"no-cache\": {}},\n                            \"content-type\": {\n                                \"application/json\": {},\n                            },\n                            \"client-request-id\": {\n                                \"061ef776-ae17-49c6-bbac-b0d62d7d9704\": {},\n                            },\n                            \"strict-transport-security\": {\n                                \"max-age=31536000\": {},\n                            },\n                        },\n                    },\n                },\n            },\n            backingStore: \u003c*store.InMemoryBackingStore | 0xc000e253c0\u003e{\n                returnOnlyChangedValues: false,\n                initializationCompleted: false,\n                store: {\n                    \"errorEscaped\": \u003c*odataerrors.MainError | 0xc000d26ce0\u003e{\n                        backingStore: \u003c*store.InMemoryBackingStore | 0xc000e253e0\u003e{\n                            returnOnlyChangedValues: false,\n                            initializationCompleted: false,\n                            store: {\n                                \"code\": \u003c*string | 0xc000d26b60\u003e\"Request_ResourceNotFound\",\n                                \"message\": \u003c*string | 0xc000d26bb0\u003e\"Resource '02691b9b-c5f7-45b0-9cff-7fc2c9805fcd' does not exist or one of its queried reference-property objects are not present.\",\n                                \"innerError\": \u003c*odataerrors.InnerError | 0xc000d26d40\u003e{\n                                    backingStore: \u003c*store.InMemoryBackingStore | 0xc000e25400\u003e{\n                                        returnOnlyChangedValues: false,\n                                        initializationCompleted: false,\n                                        store: {...: ..., ...: ...},\n                                        subscribers: {},\n                                        changedValues: {...: ..., ...: ...},\n                                    },\n                                },\n                            },\n                            subscribers: {},\n                            changedValues: {\"code\": false, \"message\": false, \"innerError\": false},\n                        },\n                    },\n                },\n                subscribers: {},\n                changedValues: {\n                    \"errorEscaped\": false,\n                },\n            },\n        },\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038798596739960832","occurred_at":"2026-03-31T02:00:39Z","signature_id":"dcd139fb1ebeeddda77d0590aec0556e86c5a78307a2bbe487575005cdbd94b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2038765373188214784","occurred_at":"2026-03-30T23:48:38Z","signature_id":"cc2bccb2a2b60ce37519536cdbed9cc7423921a24ea7ed594f472686dfb158ad","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"1663595c725a539d4430499c75496841a6e725e86c061ab925a6870046e0606e","failure_pattern":"UpdateHCPCluster20251223 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-j9bcfb\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-j9bcfb, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:231]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008ee840\u003e: \n    failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-j9bcfb\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-j9bcfb, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-j9bcfb\\\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-j9bcfb, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0008ba000\u003e{\n                s: \"timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-j9bcfb\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:231]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008ee840\u003e: \n    failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-j9bcfb\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-j9bcfb, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-j9bcfb\\\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-j9bcfb, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0008ba000\u003e{\n                s: \"timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-j9bcfb\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2040248180829851648","occurred_at":"2026-04-04T02:00:46Z","signature_id":"79be309de5c8ad4c6e57a1f38af62709c7522c3d70c281827d02832ac568702a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"874f7f16a0de76992185fc0a72b94dc6fc102bf27a7c4b1aa28301f1025b5686","failure_pattern":"found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-8mw26m","search_query":"occurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":5,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,1,0,0,0,0],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246]: Unexpected error:\n    \u003c*errors.joinError | 0xc00161a870\u003e: \n    found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-8mw26m\n    {\n        errs: [\n            \u003c*errors.errorString | 0xc001950750\u003e{\n                s: \"found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-8mw26m\",\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246]: Unexpected error:\n    \u003c*errors.joinError | 0xc00161a870\u003e: \n    found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-8mw26m\n    {\n        errs: [\n            \u003c*errors.errorString | 0xc001950750\u003e{\n                s: \"found 1 managed resource groups left behind HCP clusters in rg-cluster-nsg-subnet-reuse-8mw26m\",\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2039475678092464128","occurred_at":"2026-04-01T22:51:07Z","signature_id":"efe40ef3c88aaa17efe2662e93d8b30c200b0134f9de3ca8fce697b912e390a4","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}