{"meta":{"start_date":"2026-03-30","end_date":"2026-04-05","start_at":"2026-03-30T00:00:00Z","end_at":"2026-04-06T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T15:42:05Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":34,"failed_runs":23,"raw_occurrences":196,"matched_occurrences":196,"runs_affected":23},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-bcxzkh, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-bcxzkh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-bcxzkh, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":43,"runs_affected":5,"failed_runs":5,"run_impact_percent":14.705882352941176,"anchor_occurrences":43,"after_last_push_count":66,"trend_counts":[0,8,2,0,12,0,0,21],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-02","2026-03-16","2026-03-23"],"prior_runs_affected":19,"prior_last_seen_at":"2026-03-28T02:00:14Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a712f0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-x5gxtd\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-x5gxtd\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f71180\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd\",\n },\n \u003c*fmt.wrapErrors | 0xc000a712c0\u003e{\n msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-x5gxtd\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f71180\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a712f0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-x5gxtd\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-x5gxtd\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f71180\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd\",\n },\n \u003c*fmt.wrapErrors | 0xc000a712c0\u003e{\n msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-x5gxtd\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f71180\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000ff2060\u003e: \n failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-bg5ssk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk, error: failed waiting for cluster=\\\"ephemeral-disk\\\" in resourcegroup=\\\"ephemeral-osdisk-bg5ssk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0016a4970\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk\",\n },\n \u003c*fmt.wrapErrors | 0xc000ff2030\u003e{\n msg: \"failed waiting for cluster=\\\"ephemeral-disk\\\" in resourcegroup=\\\"ephemeral-osdisk-bg5ssk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0016a4970\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000ff2060\u003e: \n failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-bg5ssk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk, error: failed waiting for cluster=\\\"ephemeral-disk\\\" in resourcegroup=\\\"ephemeral-osdisk-bg5ssk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0016a4970\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk\",\n },\n \u003c*fmt.wrapErrors | 0xc000ff2030\u003e{\n msg: \"failed waiting for cluster=\\\"ephemeral-disk\\\" in resourcegroup=\\\"ephemeral-osdisk-bg5ssk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0016a4970\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f36a20\u003e: \n failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-29zthj\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-29zthj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013c5050\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj\",\n },\n \u003c*fmt.wrapErrors | 0xc000f369f0\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-29zthj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013c5050\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f36a20\u003e: \n failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-29zthj\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-29zthj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013c5050\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj\",\n },\n \u003c*fmt.wrapErrors | 0xc000f369f0\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-29zthj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013c5050\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"00ab9f349c37cace44ef737b7f39f910886c1bd8ba9676b0562f034c25a8ae33","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"053295b4c9bb34ced8b4896cb5f04b2fb7aaf213b0d436fc0689fa61dec430bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"23e0e96a24bf50926d4729671f18c647fe8bccd730076b6fff5ae412e8338207","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"350b1a62e1342c6815b46296bd6437dcd7493df89e7393e8433a2393c3c9bff9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"39755feb11972b185fc42575607b4e8689262016720153c9ce24fcc0f8a860d7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"45eccc39c15d47eb1ae57bc3247d60021a5ecbc41968bf40443218fc36a668e6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"4aaae7898bf5885a2613421fd344ab39a18ff9e3e8ab8d8011cfe5d8ed2d03bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"4c6b3b7b0a7eaba966864a1fb887aab5dfa1131ec7b367a4659524bb2c977fab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"4e8b4726a81c90f9e536d4e9195bd043a13ec61db4e00466820d97eedb034c1e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"51b1f572f3064bf9c7139c24f7a5aaf771e4a4007b330e0825b3e846f8d1607e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"609551c66757e23b1e38e4506ea8aa431a5c6c5469021c37bb52917afb83c2cb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"651bdce25d88a5f50c9ac1933e919c5be63488849cb55ae559854b7a33f12373","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"81b4a683f1792ba574f354cd895a4e18e1118efbd4afb62c9d57d3bd8707f44b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"8b3e01d5201b5325161b92be828648d9db5a1768a5790f9ef9547fad54fc5252","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"8de6492a8c56e32f398355194797f58a48d0f5449092d48634977b569dd67283","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"9c5b6e9aae412cc03fa41491323931ce98ae35f654258ee646f666161fb77699","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"a69b2cdd4c6d758036eaed47353b17321825e75e6ba567bd9d1329587b752223","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"cdaae32d0b4d5da24b1868adc971d200c387fb2f9c70c1d60e65fe874fecaa6e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"e16a8099164fd4a7b4a38e2bdfbcbbd008f2cf4ded26e31eef62089cbdd86f7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"e3157a6383d92e772f1ff06f4db30232d7e03c7c2732d7f7b979ce36d501e6d3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"f10b1a5e6cd97290a6c5c185969030c2102618e145d5e081ea3f695ce5196a35","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"0712c50b4b58d80c7fa67f9c8b147184116e65cd92f9e399e67019ad4489ec1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"0c557fcf018f9a61194a38b090f24f00dd3e276e1f2d7cfd13ebc026187bd706","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"28ca8e03fd5320e3f8efb638a095e3b0dd77134ad11a7b771536a46c0242eb16","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"5b24d397eafd651742199dea48c22fa110f454db09b76c3e8ee7360cd5472994","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"6d289eb97cdf8c771abc718ea433d15081de7b5463711d7aa5396be1ace3c0a4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"88e82f944c139e6cc19b557b3a082418af51be333b6988578d68f6c17833d85a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"8c013b50bbb0e285088843f7ca56a5d04d08e5252671dcc9fa791d1c881f1bb6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"a7cfda201a2b66ae296ad118419320731cc0627a8536ef129427b5ac3d420469","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"bc84585441cba890625bb3af9f5f77ae221485dd55718ed23036a13e579acb7c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"c6bc5c31aec45d33666384710b83a3f2ff5b4f5da157f7118eaf6285453023bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"d77a128e9ccb342aa703c1ffc152a70ad5d80aaab8aaba6ca7703762895ebdcf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"e63c24cd1d8891d1ce96daa3f00c9145ffedccd612aa864496c0aca698a66afe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038798596630908928","occurred_at":"2026-03-31T02:00:39Z","signature_id":"405986c2270dd2c9a58a9eefe9a0d0c8d508931c1b2989d0ffe4c96875a78179","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038798596630908928","occurred_at":"2026-03-31T02:00:39Z","signature_id":"f802eb932347a79c5b8c8ef86733787164f1458619a6ece3d088789502fac787","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"07dd99102ef0c96a316ea06bb5b2b17b9c88342bcecbf1509023a86856b98fb9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"0f8a161217c14d70f947ef10c126b5e2e0c7536543d2330cc1ec990702a3f4a9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"2201fcb737adff1788e343bdf5ed5cb9ad3297eae13660d83921b24e0a8529e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"282b429e195d66ba7aced1d9e59aed8c10651b9d004d6f1da945ee3827e9723d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"5f69da1f4a185e32e44fee587d59e3fc4a229d47183fe89ce3751b7596f8d7fe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"b64a21eacb8b60ac73a47cb37d1add24876868987b768a3d8b7eec5f2db99f7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"e816e50fe0ece7bf729bba3e1a1db6fed5213766ab03950651864392b8b95c88","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038417864989347840","occurred_at":"2026-03-30T00:47:45Z","signature_id":"c000b3ca5d3f8d4c1d7cca459590649f6e938868c79daa05a83e1caa53857ef1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wntpm\" in resourcegroup=\"admin-credential-lifecycle-test-5h6742\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":40,"runs_affected":8,"failed_runs":8,"run_impact_percent":23.529411764705884,"anchor_occurrences":40,"after_last_push_count":41,"trend_counts":[0,0,0,0,22,18,0,0],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-02","2026-03-23"],"prior_runs_affected":11,"prior_last_seen_at":"2026-03-27T04:16:09Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":6},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:126]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000eb6a20\u003e: \n failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-l56s4b\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-l56s4b, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-l56s4b\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-l56s4b, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000905dd0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-l56s4b\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:126]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000eb6a20\u003e: \n failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-l56s4b\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-l56s4b, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-l56s4b\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-l56s4b, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000905dd0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-l56s4b\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011b0810\u003e: \n failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-2v4phv\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-2v4phv, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-2v4phv\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-2v4phv, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009a5550\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-2v4phv\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011b0810\u003e: \n failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-2v4phv\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-2v4phv, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-2v4phv\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-2v4phv, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009a5550\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-2v4phv\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0013f0840\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-19-f4hz5d\" in resourcegroup=\"rg-zstream-upgrade-4-19-mf7hcn\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-f4hz5d in resource group rg-zstream-upgrade-4-19-mf7hcn, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-f4hz5d\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-mf7hcn\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-f4hz5d in resource group rg-zstream-upgrade-4-19-mf7hcn, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0015a34a0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-f4hz5d in resource group rg-zstream-upgrade-4-19-mf7hcn\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0013f0840\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-19-f4hz5d\" in resourcegroup=\"rg-zstream-upgrade-4-19-mf7hcn\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-f4hz5d in resource group rg-zstream-upgrade-4-19-mf7hcn, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-f4hz5d\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-mf7hcn\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-f4hz5d in resource group rg-zstream-upgrade-4-19-mf7hcn, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0015a34a0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-f4hz5d in resource group rg-zstream-upgrade-4-19-mf7hcn\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"978b5249588852e82b6e0319ffa1c4ff491d36b8733adfe165bd31b6a37ca120","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"cddfa084aa297044efcb077e7e99863ef8707f89d1167c5e6999483547025bd0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"f54f60621797a03b537810c47c073ed6e6a3b5a9e129109970782f85585da577","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040090253817024512","occurred_at":"2026-04-03T15:33:14Z","signature_id":"a101d653698556fe26c5748c10a638d54b4b0d1786393818b489710e7457d0e2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","signature_id":"13c51be1381e0dcdda4cd451026a1c0c513d2b1c12356428de97fbb810adac22","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","signature_id":"2d171a77f2efb9edea4c7031a4c5d4e19f877ce189da05ccb5c22454d8e0dc36","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","signature_id":"6aed24b9937e32b551ffc5f89291baa02e68d2ab1f1955dc02cb3cedc3b73787","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","signature_id":"9068b60dd134a34b45e2c4ca4bac89861a7aff0af4a58c254886eadfddc1f236","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","signature_id":"9623212ee8ee787df04708ce9e32276e48e27d5b6b6f225261f64c8fa2982fe4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","signature_id":"a30104af4d916dd7cbfec3de76f31032165c563f553ec6306160c9eed3e1a2b9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","signature_id":"1798b4cb1a37816e867fca622bd7878502beca3293e01e7628bd7340f11a75c8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","signature_id":"2db8f4dcd8799aa52814f81458b5d3d6bfe958d2aabd8c10d467e2405ffd0cbf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","signature_id":"501ec29b0bb62d873bf180fc4250d995a59d71226a623c67a7cb58747dd86494","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","signature_id":"90c9cdb50ab4b9b54916bc8225453c71c231afe899393e6702c0830fcf5fda85","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","signature_id":"a8d8ea4155e25db792b9e78559057064deaac8c45a6fd201e17b15a080b1a1dc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","signature_id":"c514ce210dc57018167504b68244b55a840f03fd60d71cec6b16d5a3957651d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","signature_id":"cc9896340c43f9cc28fd732671e71c508eb3570fc2132dd3a1db1ed99255adad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","signature_id":"e1fcf0aa66a65d43263fd7eb810b789a04625b5e903e70a78d19b18184a6fe25","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"0a5a5c3cd727714b958f5d99c98b66e348e3db4111454dec8b1404698f2e0794","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"602da1c819b7275183aaa1febbf8be89b01df162a3578abadddbbccd46f2c239","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"6621bf1de0d69411a9065b77e5668e42eef503bb9ffcc29837584f3701bf7719","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"77bc18f4976eeb2ac0f4331febecd092cfe9f711cc268f53af542f3b3632ccb2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"a54f1c852f9da1d8f55d2f99b680bffc752682b546f9f17e7fc3eb3655aef597","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"fb486152deca0eba09a175d7862f2d749ccb1ae46463ef20cabad4e7df2b2aaa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","signature_id":"61a0a590b848614b037b56bafbe413c7ebc46d25f3f5fd367783a83492aabae1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","signature_id":"70c3aed9dcf9d4690d316f75b5f6293fb9b255dea6bc67906dc9905abcacb821","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","signature_id":"8bb7de9b10491a5ec457f50056f87d1f63a31c9cc3e80935e93e3ba4023e2164","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","signature_id":"95d51df2e9bb8e2ab7173a4b7e98f477d5bdd1867f55a5914ac8a5bcf2017b4e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","signature_id":"a2c853e8faaf6b186309d06a6136d2f6b89badfa8f6172ae61fe9adc0c1458c0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","signature_id":"c29effccab9c07c6d8bad7280555e5a4daafa065e1f598a5672df600d11435fb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","signature_id":"f2e9ab95d59de09cd7c68b9d45603f48dcc7ae70fd06bf026e6a73c00e33f05c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","signature_id":"f41b3019192128e043d558d99fb921bcaaf70089e62332adee984faa6ebca7e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","signature_id":"3994952cbbf87a51591ede9e6c056fa912c759e66404e2786708b117d1988eb5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","signature_id":"44c962b6b7ee4eda614e728b25fe72f7f819bd96a0eafa86267710f755d3954d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","signature_id":"54c9710ba8898fedef07e0e373e1e9bec177d9d7a3a262f4872e5a0e23e67e76","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","signature_id":"a5acc838efa40f2e3015b2f5656215fc959732bf821c4f98925ba8e8460b95e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","signature_id":"a693b54b61382ae49d313d16b61863448c5c697a2d7e8a751f27a72c483613c4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039782869789839360","occurred_at":"2026-04-02T19:11:48Z","signature_id":"17372c8fccbca8cace580c5893ad858a6d2524fe5aee1dcf6598b8835c3a2b20","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039782869789839360","occurred_at":"2026-04-02T19:11:48Z","signature_id":"3dbc76b005b4e15f86bd5dc3fdc4f60aceb0cbe928754717e37b1fa61707d06e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039782869789839360","occurred_at":"2026-04-02T19:11:48Z","signature_id":"96ca622f1e934bff69e33dc1c686d3fd304f04accb666698f76f6d12378d24f4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":36,"runs_affected":18,"failed_runs":18,"run_impact_percent":52.94117647058823,"anchor_occurrences":36,"after_last_push_count":41,"trend_counts":[0,1,0,0,18,8,0,9],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-02","2026-03-09","2026-03-16"],"prior_runs_affected":30,"prior_last_seen_at":"2026-03-19T02:48:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:184]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:184]: Interrupted by User","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:42]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:42]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040852983817703424","occurred_at":"2026-04-05T18:04:03Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040610572545822720","occurred_at":"2026-04-05T02:00:47Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040090253817024512","occurred_at":"2026-04-03T15:33:14Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039885780922404864","occurred_at":"2026-04-03T02:00:44Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039774495035625472","occurred_at":"2026-04-02T18:38:31Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039774495035625472","occurred_at":"2026-04-02T18:38:31Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039766803189600256","occurred_at":"2026-04-02T18:07:57Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039766803189600256","occurred_at":"2026-04-02T18:07:57Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039741904471461888","occurred_at":"2026-04-02T16:29:01Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039741904471461888","occurred_at":"2026-04-02T16:29:01Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039734796569022464","occurred_at":"2026-04-02T16:00:42Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039734796569022464","occurred_at":"2026-04-02T16:00:42Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039733603084013568","occurred_at":"2026-04-02T15:56:01Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039733603084013568","occurred_at":"2026-04-02T15:56:01Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039651240933068800","occurred_at":"2026-04-02T10:28:45Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039651240933068800","occurred_at":"2026-04-02T10:28:45Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039651240933068800","occurred_at":"2026-04-02T10:28:45Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039651240933068800","occurred_at":"2026-04-02T10:28:45Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","occurrences":27,"runs_affected":9,"failed_runs":9,"run_impact_percent":26.470588235294116,"anchor_occurrences":27,"after_last_push_count":32,"trend_counts":[0,0,0,0,18,6,0,3],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-02","2026-03-16","2026-03-23"],"prior_runs_affected":20,"prior_last_seen_at":"2026-03-26T14:57:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","occurrences":6},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775406980 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should not be able to perform various invalid operations on cluster resources [PreLaunchSetup:None, Positivity:Negative, Importance:Medium, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:41\n STEP: creating a resource group @ 04/05/26 16:40:23.88\n \"ts\"=\"2026-04-05 16:40:23.880519\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-negative-tests-25748m\"\n STEP: creating cluster parameters @ 04/05/26 16:40:28.315\n STEP: creating customer resources @ 04/05/26 16:40:28.315\n STEP: creating the HCP cluster @ 04/05/26 16:41:24.604\n \"ts\"=\"2026-04-05 16:41:24.604958\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"negative-tests-cluster\" \"resourceGroup\"=\"rg-negative-tests-25748m\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775406980 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should not be able to perform various invalid operations on cluster resources [PreLaunchSetup:None, Positivity:Negative, Importance:Medium, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:41\n STEP: creating a resource group @ 04/05/26 16:40:23.88\n \"ts\"=\"2026-04-05 16:40:23.880519\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-negative-tests-25748m\"\n STEP: creating cluster parameters @ 04/05/26 16:40:28.315\n STEP: creating customer resources @ 04/05/26 16:40:28.315\n STEP: creating the HCP cluster @ 04/05/26 16:41:24.604\n \"ts\"=\"2026-04-05 16:41:24.604958\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"negative-tests-cluster\" \"resourceGroup\"=\"rg-negative-tests-25748m\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775404643 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to update node pool labels and taints [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:39\n STEP: creating a resource group @ 04/05/26 16:40:32.119\n \"ts\"=\"2026-04-05 16:40:32.119996\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-labels-taints-cbk9kk\"\n STEP: creating cluster parameters @ 04/05/26 16:40:36.148\n STEP: creating customer resources @ 04/05/26 16:40:36.148\n STEP: creating the HCP cluster @ 04/05/26 16:41:32.398\n \"ts\"=\"2026-04-05 16:41:32.399116\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"cluster-np-labels-taints\" \"resourceGroup\"=\"rg-np-labels-taints-cbk9kk\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775404643 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to update node pool labels and taints [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:39\n STEP: creating a resource group @ 04/05/26 16:40:32.119\n \"ts\"=\"2026-04-05 16:40:32.119996\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-labels-taints-cbk9kk\"\n STEP: creating cluster parameters @ 04/05/26 16:40:36.148\n STEP: creating customer resources @ 04/05/26 16:40:36.148\n STEP: creating the HCP cluster @ 04/05/26 16:41:32.398\n \"ts\"=\"2026-04-05 16:41:32.399116\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"cluster-np-labels-taints\" \"resourceGroup\"=\"rg-np-labels-taints-cbk9kk\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775404705 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231\n STEP: creating resource group @ 04/05/26 16:40:32.126\n \"ts\"=\"2026-04-05 16:40:32.126074\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-5tv55j-92x7wf\"\n STEP: creating cluster parameters at control plane version @ 04/05/26 16:40:35.94\n STEP: creating customer resources @ 04/05/26 16:40:36.059\n STEP: creating the HCP cluster with version 4.21.7 @ 04/05/26 16:41:32.412\n \"ts\"=\"2026-04-05 16:41:32.413023\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-5tv55j\" \"resourceGroup\"=\"rg-np-version-upgrade-5tv55j-92x7wf\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775404705 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231\n STEP: creating resource group @ 04/05/26 16:40:32.126\n \"ts\"=\"2026-04-05 16:40:32.126074\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-5tv55j-92x7wf\"\n STEP: creating cluster parameters at control plane version @ 04/05/26 16:40:35.94\n STEP: creating customer resources @ 04/05/26 16:40:36.059\n STEP: creating the HCP cluster with version 4.21.7 @ 04/05/26 16:41:32.412\n \"ts\"=\"2026-04-05 16:41:32.413023\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-5tv55j\" \"resourceGroup\"=\"rg-np-version-upgrade-5tv55j-92x7wf\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"72acd2594a80adde9b2dce5ad88b644dbf62c02e2363b0695e9c7e25a93449bd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"76653f7c556fac35b0a6b723d952b0de9fd9df80444614b88881532192459cf5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"c2fa2198346cf61422bd986d9ce744bde45180bf1e8b43388517312fcbacc025","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"520eafa06c89c4752bee7811e891867fb3257bfcb33fdc483d9269e770f97de2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"6359997a37ab65572d65f95e29e72f2054254ddf3a4cf4ac0c8c2fc46f0a1cda","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"a8d31a96b11e8a03efdf422a533fe81c28e318cf4353c55b406a4488dbb92f45","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040090253817024512","occurred_at":"2026-04-03T15:33:14Z","signature_id":"1dc9576b0674e0507d7233876890b3bd2e9f22bf38043e90a256f509b7f8b620","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040090253817024512","occurred_at":"2026-04-03T15:33:14Z","signature_id":"39b876f33ac69f73f9d738dea3e0601e180c615ee792257845cc5cbaeca39648","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","signature_id":"f7bc02877084b7a6fc5ec919ec45196d2dced18cbc2816a4647afaedf0634f72","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"2173f70b91c6bd7f5e4e4471684b57b09412d9d569b1fbe8d10eef8d55358155","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"3bdd4518e35aa05d1f68a1d08e5f52cff965914b56611388669abd719d32cb58","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"ff566a0f193f48a9ea966eddaa287843632a1772b7c0ae15a93934857c859a81","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","signature_id":"1ff548cb71a313175df858c7795325be9045905a3111e7e9176defc19836ede5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039782869789839360","occurred_at":"2026-04-02T19:11:48Z","signature_id":"69b97748a98721d03376b047e26a8ab738922dee3e3915026ee9110e4927c1ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039782869789839360","occurred_at":"2026-04-02T19:11:48Z","signature_id":"a24f878cc7a5f55b9e221a3806e22a12eb532f19da6bbd44a0b8d98d0ae9e30b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039766803189600256","occurred_at":"2026-04-02T18:07:57Z","signature_id":"0d6d738f471638f58110a043a102f05c2da76aaba189c4a5e7aa46163da27f0d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"10a831987a4be893a9ce87869d689cc985a10c8e1bb02e94c90780c844b62b5d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"343b69b9abdb0dbbbe18730a094b99c94da5fe39b0bfe0bc8d50516f80da0fc6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"544adc8e9ab97df3076f3902b260c61ba87e23a109d40f5068d246ae8edf5702","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"5f61202956ac87e3d93f10c8a415a8d113dfe2e5738870debb3c16c7d7ea925d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"879079e6ffdb78b0cfdd72cea6ac98ec70779604b0c6ae9831a74dc3d4e76e14","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"b4fe70556501746681ecb3c63407563373fd3957d8ba80d71df1e8acfa098cf1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"c4518cf266a9ae8b9147a4bc9108884198c498a4e0ee55b158614588b552026d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"c916d0a06bab1834b33b0882eec112cc16fb97ff309aae36491c5cdfa36ac14d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"d284e527b2816b754418e6bc1461dca78d18ee8332b2f10fc25ac8eb7a7b7695","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"f57ec76b632f5772ac7a0f8849727df031d5ca56f807171233b950f03ff6df85","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"f71f8c82e2c81fb9ce9bd2b9c751d34e51f6f04d77b8d2ae0b9f4bedf99710eb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":17,"runs_affected":7,"failed_runs":7,"run_impact_percent":20.58823529411765,"anchor_occurrences":17,"after_last_push_count":17,"trend_counts":[0,0,0,0,7,7,0,3],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:132]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000dfab80\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-xhmz6z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-xhmz6z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0014f4090\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"3a55fa98-acad-4ad6-a240-181bf4b8ec64\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"ca924b49-6e46-45a9-868d-228ae0481199\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: 07C6F1DD1B8E413285DC76D57E2C9826 Ref B: CHI261140507060 Ref C: 2026-04-05T20:51:26Z\",\n ],\n \"Date\": [\n \"Sun, 05 Apr 2026 20:51:26 GMT\",\n ],\n \"Content-Length\": [\"107\"],\n \"X-Envoy-Upstream-Service-Time\": [\"302\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260405T205126Z:ca924b49-6e46-45a9-868d-228ae0481199\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Expires\": [\"-1\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/36803a68-b504-4c65-a064-df40a0d32fe8\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000dfaaa0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"management.azu...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:132]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000dfab80\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-xhmz6z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-xhmz6z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0014f4090\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"3a55fa98-acad-4ad6-a240-181bf4b8ec64\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"ca924b49-6e46-45a9-868d-228ae0481199\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: 07C6F1DD1B8E413285DC76D57E2C9826 Ref B: CHI261140507060 Ref C: 2026-04-05T20:51:26Z\",\n ],\n \"Date\": [\n \"Sun, 05 Apr 2026 20:51:26 GMT\",\n ],\n \"Content-Length\": [\"107\"],\n \"X-Envoy-Upstream-Service-Time\": [\"302\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260405T205126Z:ca924b49-6e46-45a9-868d-228ae0481199\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Expires\": [\"-1\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/36803a68-b504-4c65-a064-df40a0d32fe8\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000dfaaa0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"management.azu...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:155]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000aa6120\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-j9zwrb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-j9zwrb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00093e060\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/6761ba3e-e3d5-4f2b-b121-1aaf18258695\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"X-Envoy-Upstream-Service-Time\": [\"158\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260405T205956Z:d0b57af8-bf86-42e3-81f6-8c47f1266249\",\n ],\n \"Date\": [\n \"Sun, 05 Apr 2026 20:59:56 GMT\",\n ],\n \"Content-Length\": [\"107\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 895D9C70FEA44594A28EC00C03902856 Ref B: CHI261140509062 Ref C: 2026-04-05T20:59:56Z\",\n ],\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"126ca117-9440-42c5-af31-856f795dd47d\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"d0b57af8-bf86-42e3-81f6-8c47f1266249\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000aa6060\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"m...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:155]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000aa6120\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-j9zwrb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-j9zwrb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00093e060\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/6761ba3e-e3d5-4f2b-b121-1aaf18258695\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"X-Envoy-Upstream-Service-Time\": [\"158\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260405T205956Z:d0b57af8-bf86-42e3-81f6-8c47f1266249\",\n ],\n \"Date\": [\n \"Sun, 05 Apr 2026 20:59:56 GMT\",\n ],\n \"Content-Length\": [\"107\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 895D9C70FEA44594A28EC00C03902856 Ref B: CHI261140509062 Ref C: 2026-04-05T20:59:56Z\",\n ],\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"126ca117-9440-42c5-af31-856f795dd47d\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"d0b57af8-bf86-42e3-81f6-8c47f1266249\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000aa6060\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"m...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:105]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00064c5a0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-d2x786/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-d2x786/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001695c50\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/37faf447-4cbb-45d8-ab1c-8f7f75feed59\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"1651\"],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"107\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260405T210827Z:1be05225-58f9-4e68-ae82-4585f065edaf\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Ms-Request-Id\": [\n \"f03a6005-7553-4eb6-9d49-9efe378f7a0f\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"1be05225-58f9-4e68-ae82-4585f065edaf\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: E4DC784654E84C74A2C16AB09A9371E1 Ref B: CHI261140508023 Ref C: 2026-04-05T21:08:25Z\",\n ],\n \"Date\": [\n \"Sun, 05 Apr 2026 21:08:26 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00064c380\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:105]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00064c5a0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-d2x786/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-d2x786/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001695c50\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/37faf447-4cbb-45d8-ab1c-8f7f75feed59\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"1651\"],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"107\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260405T210827Z:1be05225-58f9-4e68-ae82-4585f065edaf\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Ms-Request-Id\": [\n \"f03a6005-7553-4eb6-9d49-9efe378f7a0f\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"1be05225-58f9-4e68-ae82-4585f065edaf\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: E4DC784654E84C74A2C16AB09A9371E1 Ref B: CHI261140508023 Ref C: 2026-04-05T21:08:25Z\",\n ],\n \"Date\": [\n \"Sun, 05 Apr 2026 21:08:26 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00064c380\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040883810354073600","occurred_at":"2026-04-05T20:06:32Z","signature_id":"2d571bf651d0b8b4ee2843d8b226be03bddc79dec5efb9d4adef63867854ffa5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040883810354073600","occurred_at":"2026-04-05T20:06:32Z","signature_id":"38f51de79f43de40ad011679276e67513180cba797fb61bce2ce7134385db000","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040883810354073600","occurred_at":"2026-04-05T20:06:32Z","signature_id":"fd11db3f9cc963290c8c4f6ce44396e71a79bb480e4cfa44fbe51c0f78f68a27","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"c193d6ca0bf762e1335e6248bbfaedb1513e9e907031106da6506fe9e3ae11b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"c225061e4e181c2491304d3cbd154d4d968c63217496bd6f9e38d71d74b4ea92","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"c56aa4a27cb369c16d605912ab2f8d6cfd883cef06344a11b5dd1d06a710ba17","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040090253817024512","occurred_at":"2026-04-03T15:33:14Z","signature_id":"00a576e27987d92bd608669bdce0f6021d525d41d8ac0bd32cc3d4b74b822638","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040090253817024512","occurred_at":"2026-04-03T15:33:14Z","signature_id":"0592517f8993dd48a7d28ecb3a66d91e3cacbcb2d1ba137fadd1f4b1499b1c5d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","signature_id":"7575214c43750b29ac611602bb7070d16644923e94908c9a24d6a80a30d55228","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","signature_id":"848faab63125a0547ef1d80afb4f94d0f38adeebf4bc99e16f7dcb85fa48d55a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"0d24c8a507e608049ef5aef85ec5ea1a9f166d23aa217ff6cdbbc3789165c604","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"a0048cee302965dfeb513ef4fabde7392fc0ee3edd122e97e6c485f5ee38f30f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"bb803c8293f9cd63e51d5b6e61986b87a54a6d5bac398af30f1214398c7c2f83","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","signature_id":"49af8d5d6289c5f9653a0cd5b17556a7b822922c916541f0d601e3913728bc43","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","signature_id":"4c7844e62748bc91078d395584d560c1e7d7170e1afc3d5c2015d760a9626096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","signature_id":"d478f3c070bdbbdf6cce6725f9b35024fea57ed2435ef8346a8feef92e7f5381","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","signature_id":"f1e0d24bd3fa73c5580d0a8e6922db4fef593d3a6d46793c243e5e71c5f53b67","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"8a0eb43d82c81f5ac221f93e261d7d3906eccf9364890faef15f25844bdadb79","failure_pattern":"ERROR CODE: InternalServerError; detail message insufficient public IP address quota: required \u003ccount\u003e, available \u003ccount\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","occurrences":14,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.9411764705882355,"anchor_occurrences":14,"after_last_push_count":32,"trend_counts":[0,14,0,0,0,0,0,0],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-02","2026-03-16"],"prior_runs_affected":4,"prior_last_seen_at":"2026-03-18T21:39:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e0b7c0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-l2s4f2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9df063e-f124-4f77-8a25-358c9d5cde02\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9df063e-f124-4f77-8a25-358c9d5cde02\",\n \"name\": \"f9df063e-f124-4f77-8a25-358c9d5cde02\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-30T06:16:32.010035382Z\",\n \"endTime\": \"2026-03-30T06:16:48.681254232Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-l2s4f2\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9df063e-f124-4f77-8a25-358c9d5cde02\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9df063e-f124-4f77-8a25-358c9d5cde02\\\",\\n \\\"name\\\": \\\"f9df063e-f124-4f77-8a25-358c9d5cde02\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-30T06:16:32.010035382Z\\\",\\n \\\"endTime\\\": \\\"2026-03-30T06:16:48.681254232Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e0b7a0\u003e{\n msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-l2s4f2\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9df063e-f124-4f77-8a25-358c9d5cde02\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9df063e-f124-4f77-8a25-358c9d5cde02\\\",\\n \\\"name\\\": \\\"f9df063e-f124-4f77-8a25-358c9d5cde02\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-30T06:16:32.010035382Z\\\",\\n \\\"endTime\\\": \\\"2026-03-30T06:16:48.681254232Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00104d650\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260330T061652Z:5fa8b3d2-fffd-493a-9339-7cb51375d257\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Content-Length\": [\"503\"],\n \"X-Envoy-Upstream-Service-Time\": [\"9\"],\n \"Date\": [\n \"Mon, 30 Mar 2026 06:16:51 GMT\",\n ],\n \"Expires\": [\"-1\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 9C0B547CA14B41D4A16FDC03FE815989 Ref B: CHI261140514036 Ref C: 2026-03-30T06:16:52Z\",\n ],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e0b7c0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-l2s4f2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9df063e-f124-4f77-8a25-358c9d5cde02\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9df063e-f124-4f77-8a25-358c9d5cde02\",\n \"name\": \"f9df063e-f124-4f77-8a25-358c9d5cde02\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-30T06:16:32.010035382Z\",\n \"endTime\": \"2026-03-30T06:16:48.681254232Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-l2s4f2\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9df063e-f124-4f77-8a25-358c9d5cde02\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9df063e-f124-4f77-8a25-358c9d5cde02\\\",\\n \\\"name\\\": \\\"f9df063e-f124-4f77-8a25-358c9d5cde02\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-30T06:16:32.010035382Z\\\",\\n \\\"endTime\\\": \\\"2026-03-30T06:16:48.681254232Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000e0b7a0\u003e{\n msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-l2s4f2\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9df063e-f124-4f77-8a25-358c9d5cde02\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f9df063e-f124-4f77-8a25-358c9d5cde02\\\",\\n \\\"name\\\": \\\"f9df063e-f124-4f77-8a25-358c9d5cde02\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-30T06:16:32.010035382Z\\\",\\n \\\"endTime\\\": \\\"2026-03-30T06:16:48.681254232Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00104d650\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260330T061652Z:5fa8b3d2-fffd-493a-9339-7cb51375d257\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Content-Length\": [\"503\"],\n \"X-Envoy-Upstream-Service-Time\": [\"9\"],\n \"Date\": [\n \"Mon, 30 Mar 2026 06:16:51 GMT\",\n ],\n \"Expires\": [\"-1\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 9C0B547CA14B41D4A16FDC03FE815989 Ref B: CHI261140514036 Ref C: 2026-03-30T06:16:52Z\",\n ],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a0b040\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-t4849k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/671489be-ab4c-40e2-81b0-625f59cf6395\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/671489be-ab4c-40e2-81b0-625f59cf6395\",\n \"name\": \"671489be-ab4c-40e2-81b0-625f59cf6395\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-30T06:22:24.890658519Z\",\n \"endTime\": \"2026-03-30T06:22:55.769894295Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-t4849k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/671489be-ab4c-40e2-81b0-625f59cf6395\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/671489be-ab4c-40e2-81b0-625f59cf6395\\\",\\n \\\"name\\\": \\\"671489be-ab4c-40e2-81b0-625f59cf6395\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-30T06:22:24.890658519Z\\\",\\n \\\"endTime\\\": \\\"2026-03-30T06:22:55.769894295Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a0b020\u003e{\n msg: \"failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-t4849k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/671489be-ab4c-40e2-81b0-625f59cf6395\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/671489be-ab4c-40e2-81b0-625f59cf6395\\\",\\n \\\"name\\\": \\\"671489be-ab4c-40e2-81b0-625f59cf6395\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-30T06:22:24.890658519Z\\\",\\n \\\"endTime\\\": \\\"2026-03-30T06:22:55.769894295Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0011f0ed0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"X-Envoy-Upstream-Service-Time\": [\"9\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/be562cea-8623-4678-adaa-0057ff53bbd9\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"8b2df379-7ba3-4320-8e92-21a566e40b6c\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: BC420E598D454882AFE3F4518AA17A00 Ref B: CHI261140512052 Ref C: 2026-03-30T06:22:55Z\",\n ],\n \"Content-Length\": [\"503\"],\n \"X-Ms-Request-Id\": [\n \"fef2e839-309a-4189-8dfd-f18f57fe8ccf\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n \"X-Ms-Ratelimit-Rema...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:216]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a0b040\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-t4849k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/671489be-ab4c-40e2-81b0-625f59cf6395\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/671489be-ab4c-40e2-81b0-625f59cf6395\",\n \"name\": \"671489be-ab4c-40e2-81b0-625f59cf6395\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-30T06:22:24.890658519Z\",\n \"endTime\": \"2026-03-30T06:22:55.769894295Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-t4849k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/671489be-ab4c-40e2-81b0-625f59cf6395\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/671489be-ab4c-40e2-81b0-625f59cf6395\\\",\\n \\\"name\\\": \\\"671489be-ab4c-40e2-81b0-625f59cf6395\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-30T06:22:24.890658519Z\\\",\\n \\\"endTime\\\": \\\"2026-03-30T06:22:55.769894295Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000a0b020\u003e{\n msg: \"failed waiting for cluster=\\\"oidc-wi-cluster\\\" in resourcegroup=\\\"oidc-wi-t4849k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/671489be-ab4c-40e2-81b0-625f59cf6395\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/671489be-ab4c-40e2-81b0-625f59cf6395\\\",\\n \\\"name\\\": \\\"671489be-ab4c-40e2-81b0-625f59cf6395\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-30T06:22:24.890658519Z\\\",\\n \\\"endTime\\\": \\\"2026-03-30T06:22:55.769894295Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0011f0ed0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"X-Envoy-Upstream-Service-Time\": [\"9\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/be562cea-8623-4678-adaa-0057ff53bbd9\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"8b2df379-7ba3-4320-8e92-21a566e40b6c\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: BC420E598D454882AFE3F4518AA17A00 Ref B: CHI261140512052 Ref C: 2026-03-30T06:22:55Z\",\n ],\n \"Content-Length\": [\"503\"],\n \"X-Ms-Request-Id\": [\n \"fef2e839-309a-4189-8dfd-f18f57fe8ccf\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n \"X-Ms-Ratelimit-Rema...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0016131a0\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-8qw9d2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e8c3c061-19f1-4d72-b474-3fbe98891e9e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e8c3c061-19f1-4d72-b474-3fbe98891e9e\",\n \"name\": \"e8c3c061-19f1-4d72-b474-3fbe98891e9e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-30T06:18:33.483509413Z\",\n \"endTime\": \"2026-03-30T06:18:44.018298845Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-8qw9d2\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e8c3c061-19f1-4d72-b474-3fbe98891e9e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e8c3c061-19f1-4d72-b474-3fbe98891e9e\\\",\\n \\\"name\\\": \\\"e8c3c061-19f1-4d72-b474-3fbe98891e9e\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-30T06:18:33.483509413Z\\\",\\n \\\"endTime\\\": \\\"2026-03-30T06:18:44.018298845Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001613180\u003e{\n msg: \"failed waiting for cluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-8qw9d2\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e8c3c061-19f1-4d72-b474-3fbe98891e9e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e8c3c061-19f1-4d72-b474-3fbe98891e9e\\\",\\n \\\"name\\\": \\\"e8c3c061-19f1-4d72-b474-3fbe98891e9e\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-30T06:18:33.483509413Z\\\",\\n \\\"endTime\\\": \\\"2026-03-30T06:18:44.018298845Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00157d260\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Routing-Request-Id\": [\n \"CANADAEAST:20260330T061854Z:72e130e2-01fe-4e82-86ac-3810c7b0b18a\",\n ],\n \"Date\": [\n \"Mon, 30 Mar 2026 06:18:53 GMT\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"72e130e2-01fe-4e82-86ac-3810c7b0b18a\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 05A3F815056F4198B1B5D4420C306AED Ref B: CHI261140504023 Ref C: 2026-03-30T06:18:53Z\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"X-Ms-Request-Id\": [\n \"111c62...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0016131a0\u003e: \n failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-8qw9d2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e8c3c061-19f1-4d72-b474-3fbe98891e9e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e8c3c061-19f1-4d72-b474-3fbe98891e9e\",\n \"name\": \"e8c3c061-19f1-4d72-b474-3fbe98891e9e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-30T06:18:33.483509413Z\",\n \"endTime\": \"2026-03-30T06:18:44.018298845Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"insufficient public IP address quota: required 2, available 0\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-8qw9d2\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e8c3c061-19f1-4d72-b474-3fbe98891e9e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e8c3c061-19f1-4d72-b474-3fbe98891e9e\\\",\\n \\\"name\\\": \\\"e8c3c061-19f1-4d72-b474-3fbe98891e9e\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-30T06:18:33.483509413Z\\\",\\n \\\"endTime\\\": \\\"2026-03-30T06:18:44.018298845Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001613180\u003e{\n msg: \"failed waiting for cluster=\\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-8qw9d2\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e8c3c061-19f1-4d72-b474-3fbe98891e9e\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/e8c3c061-19f1-4d72-b474-3fbe98891e9e\\\",\\n \\\"name\\\": \\\"e8c3c061-19f1-4d72-b474-3fbe98891e9e\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-30T06:18:33.483509413Z\\\",\\n \\\"endTime\\\": \\\"2026-03-30T06:18:44.018298845Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"insufficient public IP address quota: required 2, available 0\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00157d260\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Routing-Request-Id\": [\n \"CANADAEAST:20260330T061854Z:72e130e2-01fe-4e82-86ac-3810c7b0b18a\",\n ],\n \"Date\": [\n \"Mon, 30 Mar 2026 06:18:53 GMT\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"72e130e2-01fe-4e82-86ac-3810c7b0b18a\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"249\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3749\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 05A3F815056F4198B1B5D4420C306AED Ref B: CHI261140504023 Ref C: 2026-03-30T06:18:53Z\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"X-Ms-Request-Id\": [\n \"111c62...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"07de22c7ba067e58ef0779ba41096f1baed00df2e146db543f223e0ebdfd6018","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"199937b5a93b05525bffe190253c866a5dbf8773ddaa01587ec35d689c85a9d1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"1e0cd8fed8de990d82e692d34df7e0638d8754776fe419023763103ce64c014d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"2bbbc9475b7baa7765b6faed00b6acb11fa7dd799d11a2fc47831134a9116f55","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"5685599a666c90b638f47dac6db24bac24a110ff1420540b7b176f0cb93be37d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"5c44b1aa022bbc2f32c15d3b9b106c6d1837b2296cc24fb4c71acd713e82e030","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"69e113039c99b4606db5fc5a6cf715a48e9278e4c6e532e92e0cc24ed3781e67","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"ab83ec7febd83d8a88e3229d3892bddcba8f893f74aa7736effa1d2f5411b6b7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"ce06702b0a7d4ee7bf9257141343e2294d5b83187ed8accbab62864afb9935ba","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"dc220b20cedb113073d89972ff37f3079f1231bfd5f407dbd93362d4c306ebd6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"f41ef081f8f749d741f3f923d091406337dd48f48c6402363c65cf49eeef72d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"fa45a75f929ac438b7d2a200daacbde36ba347019c409f8b2862dc9543db8de1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"fb73900162fc8fab223bbd19e5cd3fc29c1891367bd1306fa639b3dbeb0df2c6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"ff8db3e21ed5543ef9e5fb436a63c6e1df4cdd3f840ee7c80a6f0a89d2e6435f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout","failed_at":"unknown","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":14.705882352941176,"anchor_occurrences":5,"after_last_push_count":7,"trend_counts":[0,0,0,0,2,2,0,1],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-23"],"prior_runs_affected":7,"prior_last_seen_at":"2026-03-26T14:57:05Z","contributing_tests":[{"failed_at":"unknown","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":5}],"full_error_samples":["{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-05T17:06:11Z\"}","{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-03T20:48:28Z\"}","{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-03T08:22:36Z\"}"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c36ae336b29ede965426348568977f4287cdb5ff3cdeab9c4ba7530a7df03c1a","failure_pattern":"ERROR CODE: InternalServerError; detail code AuthorizationFailed; detail message The client '\u003cuuid\u003e' with object id '\u003cuuid\u003e' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/\u003cresource\u003e' over scope '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.9411764705882355,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,3,0,0,0],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0010502c0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-8xlw7k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\",\n \"name\": \"3484e971-8275-4ae3-a76f-65d592316cf3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T01:29:14.718319727Z\",\n \"endTime\": \"2026-04-02T01:30:18.264779861Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b531933373' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-8xlw7k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\\\",\\n \\\"name\\\": \\\"3484e971-8275-4ae3-a76f-65d592316cf3\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:29:14.718319727Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:30:18.264779861Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b531933373' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0010502a0\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-8xlw7k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\\\",\\n \\\"name\\\": \\\"3484e971-8275-4ae3-a76f-65d592316cf3\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:29:14.718319727Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:30:18.264779861Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b5319333...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0010502c0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-8xlw7k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\",\n \"name\": \"3484e971-8275-4ae3-a76f-65d592316cf3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T01:29:14.718319727Z\",\n \"endTime\": \"2026-04-02T01:30:18.264779861Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b531933373' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-8xlw7k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\\\",\\n \\\"name\\\": \\\"3484e971-8275-4ae3-a76f-65d592316cf3\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:29:14.718319727Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:30:18.264779861Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b531933373' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0010502a0\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-8xlw7k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\\\",\\n \\\"name\\\": \\\"3484e971-8275-4ae3-a76f-65d592316cf3\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:29:14.718319727Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:30:18.264779861Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b5319333...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00078d420\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-p4zvhp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\",\n \"name\": \"f32ace59-a2fb-42f6-9421-b40f5132fd29\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T00:27:03.073725945Z\",\n \"endTime\": \"2026-04-02T00:27:41.510479828Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-p4zvhp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\\\",\\n \\\"name\\\": \\\"f32ace59-a2fb-42f6-9421-b40f5132fd29\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T00:27:03.073725945Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T00:27:41.510479828Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00078d3c0\u003e{\n msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-p4zvhp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\\\",\\n \\\"name\\\": \\\"f32ace59-a2fb-42f6-9421-b40f5132fd29\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T00:27:03.073725945Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T00:27:41.510479828Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not ha...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00078d420\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-p4zvhp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\",\n \"name\": \"f32ace59-a2fb-42f6-9421-b40f5132fd29\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T00:27:03.073725945Z\",\n \"endTime\": \"2026-04-02T00:27:41.510479828Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-p4zvhp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\\\",\\n \\\"name\\\": \\\"f32ace59-a2fb-42f6-9421-b40f5132fd29\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T00:27:03.073725945Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T00:27:41.510479828Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00078d3c0\u003e{\n msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-p4zvhp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\\\",\\n \\\"name\\\": \\\"f32ace59-a2fb-42f6-9421-b40f5132fd29\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T00:27:03.073725945Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T00:27:41.510479828Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not ha...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000894f20\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-8sxmq6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\",\n \"name\": \"a300f2ba-df79-4404-a4a1-aae22a4d2587\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T00:22:54.935266426Z\",\n \"endTime\": \"2026-04-02T00:23:25.288802016Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-8sxmq6\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\\\",\\n \\\"name\\\": \\\"a300f2ba-df79-4404-a4a1-aae22a4d2587\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T00:22:54.935266426Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T00:23:25.288802016Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000894f00\u003e{\n msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-8sxmq6\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\\\",\\n \\\"name\\\": \\\"a300f2ba-df79-4404-a4a1-aae22a4d2587\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T00:22:54.935266426Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T00:23:25.288802016Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authori...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000894f20\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-8sxmq6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\",\n \"name\": \"a300f2ba-df79-4404-a4a1-aae22a4d2587\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T00:22:54.935266426Z\",\n \"endTime\": \"2026-04-02T00:23:25.288802016Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-8sxmq6\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\\\",\\n \\\"name\\\": \\\"a300f2ba-df79-4404-a4a1-aae22a4d2587\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T00:22:54.935266426Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T00:23:25.288802016Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000894f00\u003e{\n msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-8sxmq6\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\\\",\\n \\\"name\\\": \\\"a300f2ba-df79-4404-a4a1-aae22a4d2587\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T00:22:54.935266426Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T00:23:25.288802016Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authori...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"760e7b83d354480bf49188911ed5c051f4b34e00e2943fc1a4213892da050339","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"8eb97e22943903cb8d934c6e5984a6ccdef7fd32962faac882fba587161198b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"bcb22337065c03471ff7a0d8c4db52818ec22b275d83d1ee92f21bc5c25bc706","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":5.882352941176471,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,1,0,0,1,0,0,0],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-02"],"prior_runs_affected":3,"prior_last_seen_at":"2026-03-04T11:18:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:153]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:153]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.9411764705882355,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-02","2026-03-16"],"prior_runs_affected":6,"prior_last_seen_at":"2026-03-18T21:39:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.004s.\nCluster should become ready within 45 minutes\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.004s.\nCluster should become ready within 45 minutes\nExpected\n \u003cbool\u003e: false\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"ee8099a3b414c84f0dabbe6ee3f745157c59a94f539cde4bae37fe8fe9711731","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d78c0bca89c95459375355a5923414dd1603fcc7a1039f0264667a1b20c77dc3","failure_pattern":"ERROR CODE: DeploymentNotFound","search_query":"ERROR CODE: DeploymentNotFound","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.9411764705882355,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:109]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0010281e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-subnet-reuse-fnhchx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-subnet-reuse-fnhchx/operationStatuses/08584264532945813091\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: DeploymentNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"DeploymentNotFound\",\n \"message\": \"Deployment 'mi-cluster-subnet-reuse-fnhchx' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-subnet-reuse-fnhchx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-subnet-reuse-fnhchx/operationStatuses/08584264532945813091\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'mi-cluster-subnet-reuse-fnhchx' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0010281a0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-cluster-subnet-reuse-fnhchx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-subnet-reuse-fnhchx/operationStatuses/08584264532945813091\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'mi-cluster-subnet-reuse-fnhchx' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001028180\u003e{\n msg: \"failed waiting for deployment \\\"mi-cluster-subnet-reuse-fnhchx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-subnet-reuse-fnhchx/operationStatuses/08584264532945813091\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'mi-cluster-subnet-reuse-fnhchx' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0006421b0\u003e{\n ErrorCode: \"DeploymentNotFound\",\n StatusCode: 404,\n RawResponse: {\n Status: \"404 Not Found\",\n StatusCode: 404,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3748\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 18:13:11 GMT\",\n ],\n \"X-Ms-Failure-Cause\": [\"gateway\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"248\"],\n \"X-Ms-Request-Id\": [\n \"4af1ec23-5e77-436c-a930-c88c039bbaf2\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"4af1ec23-5e77-436c-a930-c88c039bbaf2\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"115\"],\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:109]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0010281e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-subnet-reuse-fnhchx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-subnet-reuse-fnhchx/operationStatuses/08584264532945813091\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: DeploymentNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"DeploymentNotFound\",\n \"message\": \"Deployment 'mi-cluster-subnet-reuse-fnhchx' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-subnet-reuse-fnhchx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-subnet-reuse-fnhchx/operationStatuses/08584264532945813091\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'mi-cluster-subnet-reuse-fnhchx' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0010281a0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-cluster-subnet-reuse-fnhchx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-subnet-reuse-fnhchx/operationStatuses/08584264532945813091\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'mi-cluster-subnet-reuse-fnhchx' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001028180\u003e{\n msg: \"failed waiting for deployment \\\"mi-cluster-subnet-reuse-fnhchx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-subnet-reuse-fnhchx/operationStatuses/08584264532945813091\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'mi-cluster-subnet-reuse-fnhchx' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0006421b0\u003e{\n ErrorCode: \"DeploymentNotFound\",\n StatusCode: 404,\n RawResponse: {\n Status: \"404 Not Found\",\n StatusCode: 404,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3748\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 18:13:11 GMT\",\n ],\n \"X-Ms-Failure-Cause\": [\"gateway\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"248\"],\n \"X-Ms-Request-Id\": [\n \"4af1ec23-5e77-436c-a930-c88c039bbaf2\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"4af1ec23-5e77-436c-a930-c88c039bbaf2\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"115\"],\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039766803189600256","occurred_at":"2026-04-02T18:07:57Z","signature_id":"e39559634ab284bff6218016b0a95f285d7a8af2e09376008215d96e16037f4e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"3faf68220dd7536a7ab4746fe0c4241f0ba52074a7b568e4e5cd42f42f541cba","failure_pattern":"ERROR CODE: InternalServerError; detail code AuthorizationFailed; detail message The client '\u003cuuid\u003e' with object id '\u003cuuid\u003e' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/\u003cresource\u003e' over scope '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j' or the scope is invalid. If access was recently granted, please refresh your credentials.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.9411764705882355,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000bea800\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-z8jp8j: failed waiting for cluster=\"np-version-upgrade-cluster-z8jp8j\" in resourcegroup=\"rg-np-version-upgrade-z8jp8j-brqz27\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\",\n \"name\": \"3ad28075-f0f4-45dd-b143-f32346eb45d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T01:33:37.647438148Z\",\n \"endTime\": \"2026-04-02T01:34:08.890577638Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg-np-upgrade-z8jp8j' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b531933373' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-version-upgrade-cluster-z8jp8j: failed waiting for cluster=\\\"np-version-upgrade-cluster-z8jp8j\\\" in resourcegroup=\\\"rg-np-version-upgrade-z8jp8j-brqz27\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\\\",\\n \\\"name\\\": \\\"3ad28075-f0f4-45dd-b143-f32346eb45d0\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:33:37.647438148Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:34:08.890577638Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-np-upgrade-z8jp8j' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b531933373' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000bea7c0\u003e{\n msg: \"failed waiting for cluster=\\\"np-version-upgrade-cluster-z8jp8j\\\" in resourcegroup=\\\"rg-np-version-upgrade-z8jp8j-brqz27\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\\\",\\n \\\"name\\\": \\\"3ad28075-f0f4-45dd-b143-f32346eb45d0\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:33:37.647438148Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:34:08.890577638Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-np-upgrade-z8jp8j' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000bea800\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-z8jp8j: failed waiting for cluster=\"np-version-upgrade-cluster-z8jp8j\" in resourcegroup=\"rg-np-version-upgrade-z8jp8j-brqz27\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\",\n \"name\": \"3ad28075-f0f4-45dd-b143-f32346eb45d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T01:33:37.647438148Z\",\n \"endTime\": \"2026-04-02T01:34:08.890577638Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg-np-upgrade-z8jp8j' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b531933373' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-version-upgrade-cluster-z8jp8j: failed waiting for cluster=\\\"np-version-upgrade-cluster-z8jp8j\\\" in resourcegroup=\\\"rg-np-version-upgrade-z8jp8j-brqz27\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\\\",\\n \\\"name\\\": \\\"3ad28075-f0f4-45dd-b143-f32346eb45d0\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:33:37.647438148Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:34:08.890577638Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-np-upgrade-z8jp8j' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b531933373' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000bea7c0\u003e{\n msg: \"failed waiting for cluster=\\\"np-version-upgrade-cluster-z8jp8j\\\" in resourcegroup=\\\"rg-np-version-upgrade-z8jp8j-brqz27\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\\\",\\n \\\"name\\\": \\\"3ad28075-f0f4-45dd-b143-f32346eb45d0\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:33:37.647438148Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:34:08.890577638Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-np-upgrade-z8jp8j' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"20cc6f67b762823d979736e097ec3b236ea3bc2eaa4a0f6f645004250bc4c285","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"3be3ce8c6d012d61a6e19fef97bf2b9573e9f025baf6aedc89c2d331c4289e17","failure_pattern":"ERROR CODE: InternalServerError; detail code AuthorizationFailed; detail message The client '\u003cuuid\u003e' with object id '\u003cuuid\u003e' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/\u003cresource\u003e' over scope '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-19' or the scope is invalid. If access was recently granted, please refresh your credentials.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.9411764705882355,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000334400\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-57zxqg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c3b1341-fda3-41e9-8dd4-5c30342a2d31\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c3b1341-fda3-41e9-8dd4-5c30342a2d31\",\n \"name\": \"3c3b1341-fda3-41e9-8dd4-5c30342a2d31\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-30T06:12:39.328826001Z\",\n \"endTime\": \"2026-03-30T06:13:08.581005823Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Failed to get virtual network 'customer-vnet-name-4-19' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-57zxqg/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-19\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client '3820d679-b809-47f8-8f80-79cbd25d1d89' with object id '62b94c2a-37fd-4b0a-b97c-1370b69f6c0c' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-57zxqg/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-19' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-57zxqg\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c3b1341-fda3-41e9-8dd4-5c30342a2d31\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c3b1341-fda3-41e9-8dd4-5c30342a2d31\\\",\\n \\\"name\\\": \\\"3c3b1341-fda3-41e9-8dd4-5c30342a2d31\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-30T06:12:39.328826001Z\\\",\\n \\\"endTime\\\": \\\"2026-03-30T06:13:08.581005823Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-name-4-19' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-57zxqg/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-19\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '3820d679-b809-47f8-8f80-79cbd25d1d89' with object id '62b94c2a-37fd-4b0a-b97c-1370b69f6c0c' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-57zxqg/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-19' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001475530\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadaeast/f58e194a-3f86-4008-98dc-aa5f60802195\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 60DFA7F6143244D1924498FA54547EC7 Ref B: CHI261140510034 Ref C: 2026-03-30T06:13:10Z\",\n ],\n \"Date\": [\n \"Mon, 30 Mar 2026 06:13:10 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"bdec501d-0ca0-4486-8500-77afaa68e37a\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADAEAST:20260330T061310Z:bdec501d-0ca0-4486-8500-77afaa68e37a\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"9\"],\n \"Content-Length\": [\"1563\"],\n \"Content-Type\": [\"application/json\"],\n \"X-M...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000334400\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-57zxqg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c3b1341-fda3-41e9-8dd4-5c30342a2d31\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c3b1341-fda3-41e9-8dd4-5c30342a2d31\",\n \"name\": \"3c3b1341-fda3-41e9-8dd4-5c30342a2d31\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-03-30T06:12:39.328826001Z\",\n \"endTime\": \"2026-03-30T06:13:08.581005823Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Failed to get virtual network 'customer-vnet-name-4-19' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-57zxqg/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-19\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client '3820d679-b809-47f8-8f80-79cbd25d1d89' with object id '62b94c2a-37fd-4b0a-b97c-1370b69f6c0c' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-57zxqg/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-19' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-57zxqg\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c3b1341-fda3-41e9-8dd4-5c30342a2d31\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c3b1341-fda3-41e9-8dd4-5c30342a2d31\\\",\\n \\\"name\\\": \\\"3c3b1341-fda3-41e9-8dd4-5c30342a2d31\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-03-30T06:12:39.328826001Z\\\",\\n \\\"endTime\\\": \\\"2026-03-30T06:13:08.581005823Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-name-4-19' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-57zxqg/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-19\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '3820d679-b809-47f8-8f80-79cbd25d1d89' with object id '62b94c2a-37fd-4b0a-b97c-1370b69f6c0c' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-back-version-57zxqg/providers/Microsoft.Network/virtualNetworks/customer-vnet-name-4-19' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001475530\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 200,\n RawResponse: {\n Status: \"200 OK\",\n StatusCode: 200,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadaeast/f58e194a-3f86-4008-98dc-aa5f60802195\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 60DFA7F6143244D1924498FA54547EC7 Ref B: CHI261140510034 Ref C: 2026-03-30T06:13:10Z\",\n ],\n \"Date\": [\n \"Mon, 30 Mar 2026 06:13:10 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"bdec501d-0ca0-4486-8500-77afaa68e37a\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADAEAST:20260330T061310Z:bdec501d-0ca0-4486-8500-77afaa68e37a\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"9\"],\n \"Content-Length\": [\"1563\"],\n \"Content-Type\": [\"application/json\"],\n \"X-M...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"45e868840d657441a22e083fb27eb79d380a9b60de23b21ce110276bbb2ea209","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"2b16feefd927dd52de60b3d699308db2fee6e43cd349d94e0653b4b9fce6f419","failure_pattern":"ERROR CODE: InternalServerError; detail code AuthorizationFailed; detail message The client '\u003cuuid\u003e' with object id '\u003cuuid\u003e' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/\u003cresource\u003e' over scope '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn' or the scope is invalid. If access was recently granted, please refresh your credentials.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.9411764705882355,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00131a180\u003e: \n failed to create HCP cluster cluster-zstream-4-21-xvw7fn: failed waiting for cluster=\"cluster-zstream-4-21-xvw7fn\" in resourcegroup=\"rg-zstream-upgrade-4-21-qf98rq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\",\n \"name\": \"66f6aa6d-0e58-4cf7-a817-132dc9b1db23\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T01:34:57.397134314Z\",\n \"endTime\": \"2026-04-02T01:36:07.715905327Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Failed to get virtual network 'customer-vnet-zstream-xvw7fn' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client '0c358081-82b1-4f65-97bc-be8647a5b112' with object id 'b3c21201-bf1b-4718-b697-526dbc98e7e9' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-21-xvw7fn: failed waiting for cluster=\\\"cluster-zstream-4-21-xvw7fn\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-qf98rq\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\\",\\n \\\"name\\\": \\\"66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:34:57.397134314Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:36:07.715905327Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-zstream-xvw7fn' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '0c358081-82b1-4f65-97bc-be8647a5b112' with object id 'b3c21201-bf1b-4718-b697-526dbc98e7e9' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00131a160\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-xvw7fn\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-qf98rq\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\\",\\n \\\"name\\\": \\\"66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:34:57.397134314Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:36:07.715905327Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-zstream-xvw7fn' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00131a180\u003e: \n failed to create HCP cluster cluster-zstream-4-21-xvw7fn: failed waiting for cluster=\"cluster-zstream-4-21-xvw7fn\" in resourcegroup=\"rg-zstream-upgrade-4-21-qf98rq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\",\n \"name\": \"66f6aa6d-0e58-4cf7-a817-132dc9b1db23\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T01:34:57.397134314Z\",\n \"endTime\": \"2026-04-02T01:36:07.715905327Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Failed to get virtual network 'customer-vnet-zstream-xvw7fn' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client '0c358081-82b1-4f65-97bc-be8647a5b112' with object id 'b3c21201-bf1b-4718-b697-526dbc98e7e9' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-21-xvw7fn: failed waiting for cluster=\\\"cluster-zstream-4-21-xvw7fn\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-qf98rq\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\\",\\n \\\"name\\\": \\\"66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:34:57.397134314Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:36:07.715905327Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-zstream-xvw7fn' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '0c358081-82b1-4f65-97bc-be8647a5b112' with object id 'b3c21201-bf1b-4718-b697-526dbc98e7e9' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00131a160\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-xvw7fn\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-qf98rq\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\\",\\n \\\"name\\\": \\\"66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:34:57.397134314Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:36:07.715905327Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-zstream-xvw7fn' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"f5b50909a454a74f214ea8348c5bbbdd39c94bd8e1438791df15454e3358b666","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ee1c062c00e1e61daa37b83a9433bafe2cf1df398e225f1de7fddf080af4e874","failure_pattern":"add password: error status code received from the API","search_query":"add password: error status code received from the API","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.9411764705882355,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:130]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008802e0\u003e: \n add password: error status code received from the API\n {\n msg: \"add password: error status code received from the API\",\n err: \u003c*odataerrors.ODataError | 0xc000b91aa0\u003e{\n ApiError: {\n Message: \"\",\n ResponseStatusCode: 404,\n ResponseHeaders: {\n header: {\n headers: {\n \"strict-transport-security\": {\n \"max-age=31536000\": {},\n },\n \"client-request-id\": {\n \"609e61cb-e058-40bf-954c-a040be046529\": {},\n },\n \"content-type\": {\n \"application/json\": {},\n },\n \"x-ms-ags-diagnostic\": {\n \"{\\\"ServerInfo\\\":{\\\"DataCenter\\\":\\\"East US\\\",\\\"Slice\\\":\\\"E\\\",\\\"Ring\\\":\\\"5\\\",\\\"ScaleUnit\\\":\\\"001\\\",\\\"RoleInstance\\\":\\\"MN1PEPF00010855\\\"}}\": {},\n },\n \"x-ms-resource-unit\": {\"1\": {}},\n \"date\": {\n \"Tue, 31 Mar 2026 03:00:12 GMT\": {},\n },\n \"vary\": {\n \"Accept-Encoding\": {},\n },\n \"request-id\": {\n \"609e61cb-e058-40bf-954c-a040be046529\": {},\n },\n \"cache-control\": {\"no-cache\": {}},\n },\n },\n },\n },\n backingStore: \u003c*store.InMemoryBackingStore | 0xc000880280\u003e{\n returnOnlyChangedValues: false,\n initializationCompleted: false,\n store: {\n \"errorEscaped\": \u003c*odataerrors.MainError | 0xc0014a1650\u003e{\n backingStore: \u003c*store.InMemoryBackingStore | 0xc0008802a0\u003e{\n returnOnlyChangedValues: false,\n initializationCompleted: false,\n store: {\n \"code\": \u003c*string | 0xc0014a14d0\u003e\"Request_ResourceNotFound\",\n \"message\": \u003c*string | 0xc0014a1520\u003e\"Resource 'b5d9870c-3832-4af0-9062-5ced2786c558' does not exist or one of its queried reference-property objects are not present.\",\n \"innerError\": \u003c*odataerrors.InnerError | 0xc0014a16b0\u003e{\n backingStore: \u003c*store.InMemoryBackingStore | 0xc0008802c0\u003e{\n returnOnlyChangedValues: false,\n initializationCompleted: false,\n store: {...: ..., ...: ...},\n subscribers: {},\n changedValues: {...: ..., ...: ...},\n },\n },\n },\n subscribers: {},\n changedValues: {\"code\": false, \"message\": false, \"innerError\": false},\n },\n },\n },\n subscribers: {},\n changedValues: {\n \"errorEscaped\": false,\n },\n },\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:130]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0008802e0\u003e: \n add password: error status code received from the API\n {\n msg: \"add password: error status code received from the API\",\n err: \u003c*odataerrors.ODataError | 0xc000b91aa0\u003e{\n ApiError: {\n Message: \"\",\n ResponseStatusCode: 404,\n ResponseHeaders: {\n header: {\n headers: {\n \"strict-transport-security\": {\n \"max-age=31536000\": {},\n },\n \"client-request-id\": {\n \"609e61cb-e058-40bf-954c-a040be046529\": {},\n },\n \"content-type\": {\n \"application/json\": {},\n },\n \"x-ms-ags-diagnostic\": {\n \"{\\\"ServerInfo\\\":{\\\"DataCenter\\\":\\\"East US\\\",\\\"Slice\\\":\\\"E\\\",\\\"Ring\\\":\\\"5\\\",\\\"ScaleUnit\\\":\\\"001\\\",\\\"RoleInstance\\\":\\\"MN1PEPF00010855\\\"}}\": {},\n },\n \"x-ms-resource-unit\": {\"1\": {}},\n \"date\": {\n \"Tue, 31 Mar 2026 03:00:12 GMT\": {},\n },\n \"vary\": {\n \"Accept-Encoding\": {},\n },\n \"request-id\": {\n \"609e61cb-e058-40bf-954c-a040be046529\": {},\n },\n \"cache-control\": {\"no-cache\": {}},\n },\n },\n },\n },\n backingStore: \u003c*store.InMemoryBackingStore | 0xc000880280\u003e{\n returnOnlyChangedValues: false,\n initializationCompleted: false,\n store: {\n \"errorEscaped\": \u003c*odataerrors.MainError | 0xc0014a1650\u003e{\n backingStore: \u003c*store.InMemoryBackingStore | 0xc0008802a0\u003e{\n returnOnlyChangedValues: false,\n initializationCompleted: false,\n store: {\n \"code\": \u003c*string | 0xc0014a14d0\u003e\"Request_ResourceNotFound\",\n \"message\": \u003c*string | 0xc0014a1520\u003e\"Resource 'b5d9870c-3832-4af0-9062-5ced2786c558' does not exist or one of its queried reference-property objects are not present.\",\n \"innerError\": \u003c*odataerrors.InnerError | 0xc0014a16b0\u003e{\n backingStore: \u003c*store.InMemoryBackingStore | 0xc0008802c0\u003e{\n returnOnlyChangedValues: false,\n initializationCompleted: false,\n store: {...: ..., ...: ...},\n subscribers: {},\n changedValues: {...: ..., ...: ...},\n },\n },\n },\n subscribers: {},\n changedValues: {\"code\": false, \"message\": false, \"innerError\": false},\n },\n },\n },\n subscribers: {},\n changedValues: {\n \"errorEscaped\": false,\n },\n },\n },\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038798596630908928","occurred_at":"2026-03-31T02:00:39Z","signature_id":"497600bff911ea045bfa7733f2a938420f421f08a6dde150fcd82db14842748e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d6c1dec4d8c20fbbd7f2b826c14447c68052c707edbc8727865c6a1f97f0a0dd","failure_pattern":"failed creating deployment \"test-vm\" in resourcegroup=\"\u003cresource-group\u003e\": PUT \u003curl\u003e","search_query":"failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-s5czh5\": PUT https://management","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.9411764705882355,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:114]: Timed out after 1800.001s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0012c8180\u003e: \n failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-s5czh5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-s5czh5/providers/Microsoft.Resources/deployments/test-vm\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidTemplateDeployment\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidTemplateDeployment\",\n \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The tracking id is 'cf5d91a3-a913-4d5b-9164-bccf81eb8eee'. See inner errors for details.\",\n \"details\": [\n {\n \"code\": \"ResourceCountExceedsLimitDueToTemplate\",\n \"message\": \"Subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has a quota of 20 for resources of type PublicIpAddress with sku SkuNotSpecified. Subscription currently has 20 resources and the template contains 1 new resources of the this type which exceeds the quota. Please contact support to increase the quota for resource type PublicIpAddress\",\n \"details\": []\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed creating deployment \\\"test-vm\\\" in resourcegroup=\\\"e2e-cidr-connectivity-s5czh5\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-s5czh5/providers/Microsoft.Resources/deployments/test-vm\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidTemplateDeployment\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidTemplateDeployment\\\",\\n \\\"message\\\": \\\"The template deployment 'test-vm' is not valid according to the validation procedure. The tracking id is 'cf5d91a3-a913-4d5b-9164-bccf81eb8eee'. See inner errors for details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"ResourceCountExceedsLimitDueToTemplate\\\",\\n \\\"message\\\": \\\"Subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has a quota of 20 for resources of type PublicIpAddress with sku SkuNotSpecified. Subscription currently has 20 resources and the template contains 1 new resources of the this type which exceeds the quota. Please contact support to increase the quota for resource type PublicIpAddress\\\",\\n \\\"details\\\": []\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000f127e0\u003e{\n ErrorCode: \"InvalidTemplateDeployment\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"198\"],\n \"X-Ms-Failure-Cause\": [\"gateway\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2998\"],\n \"X-Ms-Request-Id\": [\n \"cf5d91a3-a913-4d5b-9164-bccf81eb8eee\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"cf5d91a3-a913-4d5b-9164-bccf81eb8eee\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"EASTUS2EUAP:20260330T065415Z:cf5d91a3-a913-4d5b-9164-bccf81eb8eee\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Date\": [\n \"Mon, 30 Mar 2026 06:54:14 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: FED97ACCAC0E49C1AB5718A871C140C7 Ref B: CHI261140514023 Ref C: 2026-03-30T06:54:14Z\",\n ],\n \"Content-Length\": [\"655\"],\n \"Content-Type\": [\n \"application/json; charset=utf-8\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0012c8040\u003e{\n s: \"{\\\"error\\\":{\\\"code\\\":\\\"InvalidTemplateDeployment\\\",\\\"message\\\":\\\"The template deployment 'test-vm' is not valid according to the validation procedure. The tracking id is 'cf5d91a3-a913-4d5b-9164-bccf81eb8eee'. See inner errors for details.\\\",\\\"details\\\":[{\\\"code\\\":\\\"ResourceCountExceedsLimitDueToTemplate\\\",\\\"message\\\":\\\"Subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has a quota of 20 for resources of type PublicIpAddress with sku SkuNotSpecified. Subscription currently has 20 resources and the template contains 1 new resources of the this type which exceeds the quota. Please contact support to increase the quota for resource type PublicIpAddress\\\",\\\"detai...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:114]: Timed out after 1800.001s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0012c8180\u003e: \n failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-s5czh5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-s5czh5/providers/Microsoft.Resources/deployments/test-vm\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidTemplateDeployment\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidTemplateDeployment\",\n \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The tracking id is 'cf5d91a3-a913-4d5b-9164-bccf81eb8eee'. See inner errors for details.\",\n \"details\": [\n {\n \"code\": \"ResourceCountExceedsLimitDueToTemplate\",\n \"message\": \"Subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has a quota of 20 for resources of type PublicIpAddress with sku SkuNotSpecified. Subscription currently has 20 resources and the template contains 1 new resources of the this type which exceeds the quota. Please contact support to increase the quota for resource type PublicIpAddress\",\n \"details\": []\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed creating deployment \\\"test-vm\\\" in resourcegroup=\\\"e2e-cidr-connectivity-s5czh5\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-s5czh5/providers/Microsoft.Resources/deployments/test-vm\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidTemplateDeployment\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidTemplateDeployment\\\",\\n \\\"message\\\": \\\"The template deployment 'test-vm' is not valid according to the validation procedure. The tracking id is 'cf5d91a3-a913-4d5b-9164-bccf81eb8eee'. See inner errors for details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"ResourceCountExceedsLimitDueToTemplate\\\",\\n \\\"message\\\": \\\"Subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has a quota of 20 for resources of type PublicIpAddress with sku SkuNotSpecified. Subscription currently has 20 resources and the template contains 1 new resources of the this type which exceeds the quota. Please contact support to increase the quota for resource type PublicIpAddress\\\",\\n \\\"details\\\": []\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000f127e0\u003e{\n ErrorCode: \"InvalidTemplateDeployment\",\n StatusCode: 400,\n RawResponse: {\n Status: \"400 Bad Request\",\n StatusCode: 400,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"198\"],\n \"X-Ms-Failure-Cause\": [\"gateway\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2998\"],\n \"X-Ms-Request-Id\": [\n \"cf5d91a3-a913-4d5b-9164-bccf81eb8eee\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"cf5d91a3-a913-4d5b-9164-bccf81eb8eee\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"EASTUS2EUAP:20260330T065415Z:cf5d91a3-a913-4d5b-9164-bccf81eb8eee\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Date\": [\n \"Mon, 30 Mar 2026 06:54:14 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: FED97ACCAC0E49C1AB5718A871C140C7 Ref B: CHI261140514023 Ref C: 2026-03-30T06:54:14Z\",\n ],\n \"Content-Length\": [\"655\"],\n \"Content-Type\": [\n \"application/json; charset=utf-8\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0012c8040\u003e{\n s: \"{\\\"error\\\":{\\\"code\\\":\\\"InvalidTemplateDeployment\\\",\\\"message\\\":\\\"The template deployment 'test-vm' is not valid according to the validation procedure. The tracking id is 'cf5d91a3-a913-4d5b-9164-bccf81eb8eee'. See inner errors for details.\\\",\\\"details\\\":[{\\\"code\\\":\\\"ResourceCountExceedsLimitDueToTemplate\\\",\\\"message\\\":\\\"Subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has a quota of 20 for resources of type PublicIpAddress with sku SkuNotSpecified. Subscription currently has 20 resources and the template contains 1 new resources of the this type which exceeds the quota. Please contact support to increase the quota for resource type PublicIpAddress\\\",\\\"detai...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2038498730092531712","occurred_at":"2026-03-30T06:09:05Z","signature_id":"9bf37e3b08e675b2930cf95933153a86979a616e0d3e7464c6e248d0217acb7e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"79d555cc6e5bd17f4b96af0c9e2da6a571d8241aacf20f84304c98773ef2682b","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"e2e-cilium-gpnkb5\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.9411764705882355,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,0,1],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-02"],"prior_runs_affected":3,"prior_last_seen_at":"2026-03-05T02:00:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:186]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0020085c0\u003e: \n failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"e2e-cilium-gpnkb5\" to finish creating: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"cilium-np\\\" for cluster \\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-gpnkb5\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:186]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0020085c0\u003e: \n failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"e2e-cilium-gpnkb5\" to finish creating: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"cilium-np\\\" for cluster \\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-gpnkb5\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040610572545822720","occurred_at":"2026-04-05T02:00:47Z","signature_id":"2c3fed36a98d5c8fc924d06dac17540deb5718fccf37204fe17cb23c299f17c9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"802998fa7e45e8509bd2cc7f1eb9749f5e3559041c2c929960e3ec4481056396","failure_pattern":"found 1 managed resource groups left behind HCP clusters in cluster-listing-g7w797","search_query":"occurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to list HCP clusters without node pools at the resource group level","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.9411764705882355,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-03-29..2026-04-05 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to list HCP clusters without node pools at the resource group level","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246]: Unexpected error:\n \u003c*errors.joinError | 0xc001010078\u003e: \n found 1 managed resource groups left behind HCP clusters in cluster-listing-g7w797\n {\n errs: [\n \u003c*errors.errorString | 0xc00185b800\u003e{\n s: \"found 1 managed resource groups left behind HCP clusters in cluster-listing-g7w797\",\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246]: Unexpected error:\n \u003c*errors.joinError | 0xc001010078\u003e: \n found 1 managed resource groups left behind HCP clusters in cluster-listing-g7w797\n {\n errs: [\n \u003c*errors.errorString | 0xc00185b800\u003e{\n s: \"found 1 managed resource groups left behind HCP clusters in cluster-listing-g7w797\",\n },\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","signature_id":"bfda7f6f00a6578d4c5d8c5848019b1684cf7315d6a0363d79eb51f8fca100db","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}