{"meta":{"start_date":"2026-04-05","end_date":"2026-04-05","start_at":"2026-04-05T00:00:00Z","end_at":"2026-04-06T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T16:41:42Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":4,"failed_runs":4,"raw_occurrences":39,"matched_occurrences":39,"runs_affected":4},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-bcxzkh, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-bcxzkh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-bcxzkh, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","occurrences":21,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":21,"after_last_push_count":66,"trend_counts":[8,2,0,12,0,0,21],"trend_range":"2026-03-30..2026-04-05 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-02","2026-03-16","2026-03-23"],"prior_runs_affected":19,"prior_last_seen_at":"2026-03-28T02:00:14Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a712f0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-x5gxtd\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-x5gxtd\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f71180\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd\",\n },\n \u003c*fmt.wrapErrors | 0xc000a712c0\u003e{\n msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-x5gxtd\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f71180\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a712f0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-x5gxtd\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-x5gxtd\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f71180\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd\",\n },\n \u003c*fmt.wrapErrors | 0xc000a712c0\u003e{\n msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-x5gxtd\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f71180\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-x5gxtd\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000ff2060\u003e: \n failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-bg5ssk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk, error: failed waiting for cluster=\\\"ephemeral-disk\\\" in resourcegroup=\\\"ephemeral-osdisk-bg5ssk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0016a4970\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk\",\n },\n \u003c*fmt.wrapErrors | 0xc000ff2030\u003e{\n msg: \"failed waiting for cluster=\\\"ephemeral-disk\\\" in resourcegroup=\\\"ephemeral-osdisk-bg5ssk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0016a4970\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000ff2060\u003e: \n failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-bg5ssk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk, error: failed waiting for cluster=\\\"ephemeral-disk\\\" in resourcegroup=\\\"ephemeral-osdisk-bg5ssk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0016a4970\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk\",\n },\n \u003c*fmt.wrapErrors | 0xc000ff2030\u003e{\n msg: \"failed waiting for cluster=\\\"ephemeral-disk\\\" in resourcegroup=\\\"ephemeral-osdisk-bg5ssk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0016a4970\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-bg5ssk\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f36a20\u003e: \n failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-29zthj\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-29zthj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013c5050\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj\",\n },\n \u003c*fmt.wrapErrors | 0xc000f369f0\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-29zthj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013c5050\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f36a20\u003e: \n failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-29zthj\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-29zthj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013c5050\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj\",\n },\n \u003c*fmt.wrapErrors | 0xc000f369f0\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-29zthj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013c5050\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-29zthj\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"00ab9f349c37cace44ef737b7f39f910886c1bd8ba9676b0562f034c25a8ae33","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"053295b4c9bb34ced8b4896cb5f04b2fb7aaf213b0d436fc0689fa61dec430bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"23e0e96a24bf50926d4729671f18c647fe8bccd730076b6fff5ae412e8338207","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"350b1a62e1342c6815b46296bd6437dcd7493df89e7393e8433a2393c3c9bff9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"39755feb11972b185fc42575607b4e8689262016720153c9ce24fcc0f8a860d7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"45eccc39c15d47eb1ae57bc3247d60021a5ecbc41968bf40443218fc36a668e6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"4aaae7898bf5885a2613421fd344ab39a18ff9e3e8ab8d8011cfe5d8ed2d03bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"4c6b3b7b0a7eaba966864a1fb887aab5dfa1131ec7b367a4659524bb2c977fab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"4e8b4726a81c90f9e536d4e9195bd043a13ec61db4e00466820d97eedb034c1e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"51b1f572f3064bf9c7139c24f7a5aaf771e4a4007b330e0825b3e846f8d1607e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"609551c66757e23b1e38e4506ea8aa431a5c6c5469021c37bb52917afb83c2cb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"651bdce25d88a5f50c9ac1933e919c5be63488849cb55ae559854b7a33f12373","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"81b4a683f1792ba574f354cd895a4e18e1118efbd4afb62c9d57d3bd8707f44b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"8b3e01d5201b5325161b92be828648d9db5a1768a5790f9ef9547fad54fc5252","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"8de6492a8c56e32f398355194797f58a48d0f5449092d48634977b569dd67283","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"9c5b6e9aae412cc03fa41491323931ce98ae35f654258ee646f666161fb77699","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"a69b2cdd4c6d758036eaed47353b17321825e75e6ba567bd9d1329587b752223","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"cdaae32d0b4d5da24b1868adc971d200c387fb2f9c70c1d60e65fe874fecaa6e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"e16a8099164fd4a7b4a38e2bdfbcbbd008f2cf4ded26e31eef62089cbdd86f7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"e3157a6383d92e772f1ff06f4db30232d7e03c7c2732d7f7b979ce36d501e6d3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"f10b1a5e6cd97290a6c5c185969030c2102618e145d5e081ea3f695ce5196a35","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":9,"runs_affected":3,"failed_runs":3,"run_impact_percent":75,"anchor_occurrences":9,"after_last_push_count":41,"trend_counts":[1,0,0,18,8,0,9],"trend_range":"2026-03-30..2026-04-05 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-02","2026-03-09","2026-03-16"],"prior_runs_affected":30,"prior_last_seen_at":"2026-03-19T02:48:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:184]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:184]: Interrupted by User","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:42]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:42]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040852983817703424","occurred_at":"2026-04-05T18:04:03Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040610572545822720","occurred_at":"2026-04-05T02:00:47Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":3,"after_last_push_count":32,"trend_counts":[0,0,0,18,6,0,3],"trend_range":"2026-03-30..2026-04-05 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-02","2026-03-16","2026-03-23"],"prior_runs_affected":20,"prior_last_seen_at":"2026-03-26T14:57:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775406980 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should not be able to perform various invalid operations on cluster resources [PreLaunchSetup:None, Positivity:Negative, Importance:Medium, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:41\n STEP: creating a resource group @ 04/05/26 16:40:23.88\n \"ts\"=\"2026-04-05 16:40:23.880519\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-negative-tests-25748m\"\n STEP: creating cluster parameters @ 04/05/26 16:40:28.315\n STEP: creating customer resources @ 04/05/26 16:40:28.315\n STEP: creating the HCP cluster @ 04/05/26 16:41:24.604\n \"ts\"=\"2026-04-05 16:41:24.604958\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"negative-tests-cluster\" \"resourceGroup\"=\"rg-negative-tests-25748m\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775406980 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should not be able to perform various invalid operations on cluster resources [PreLaunchSetup:None, Positivity:Negative, Importance:Medium, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:41\n STEP: creating a resource group @ 04/05/26 16:40:23.88\n \"ts\"=\"2026-04-05 16:40:23.880519\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-negative-tests-25748m\"\n STEP: creating cluster parameters @ 04/05/26 16:40:28.315\n STEP: creating customer resources @ 04/05/26 16:40:28.315\n STEP: creating the HCP cluster @ 04/05/26 16:41:24.604\n \"ts\"=\"2026-04-05 16:41:24.604958\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"negative-tests-cluster\" \"resourceGroup\"=\"rg-negative-tests-25748m\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775404643 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to update node pool labels and taints [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:39\n STEP: creating a resource group @ 04/05/26 16:40:32.119\n \"ts\"=\"2026-04-05 16:40:32.119996\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-labels-taints-cbk9kk\"\n STEP: creating cluster parameters @ 04/05/26 16:40:36.148\n STEP: creating customer resources @ 04/05/26 16:40:36.148\n STEP: creating the HCP cluster @ 04/05/26 16:41:32.398\n \"ts\"=\"2026-04-05 16:41:32.399116\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"cluster-np-labels-taints\" \"resourceGroup\"=\"rg-np-labels-taints-cbk9kk\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775404643 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to update node pool labels and taints [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:39\n STEP: creating a resource group @ 04/05/26 16:40:32.119\n \"ts\"=\"2026-04-05 16:40:32.119996\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-labels-taints-cbk9kk\"\n STEP: creating cluster parameters @ 04/05/26 16:40:36.148\n STEP: creating customer resources @ 04/05/26 16:40:36.148\n STEP: creating the HCP cluster @ 04/05/26 16:41:32.398\n \"ts\"=\"2026-04-05 16:41:32.399116\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"cluster-np-labels-taints\" \"resourceGroup\"=\"rg-np-labels-taints-cbk9kk\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775404705 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231\n STEP: creating resource group @ 04/05/26 16:40:32.126\n \"ts\"=\"2026-04-05 16:40:32.126074\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-5tv55j-92x7wf\"\n STEP: creating cluster parameters at control plane version @ 04/05/26 16:40:35.94\n STEP: creating customer resources @ 04/05/26 16:40:36.059\n STEP: creating the HCP cluster with version 4.21.7 @ 04/05/26 16:41:32.412\n \"ts\"=\"2026-04-05 16:41:32.413023\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-5tv55j\" \"resourceGroup\"=\"rg-np-version-upgrade-5tv55j-92x7wf\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775404705 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231\n STEP: creating resource group @ 04/05/26 16:40:32.126\n \"ts\"=\"2026-04-05 16:40:32.126074\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-5tv55j-92x7wf\"\n STEP: creating cluster parameters at control plane version @ 04/05/26 16:40:35.94\n STEP: creating customer resources @ 04/05/26 16:40:36.059\n STEP: creating the HCP cluster with version 4.21.7 @ 04/05/26 16:41:32.412\n \"ts\"=\"2026-04-05 16:41:32.413023\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-5tv55j\" \"resourceGroup\"=\"rg-np-version-upgrade-5tv55j-92x7wf\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"72acd2594a80adde9b2dce5ad88b644dbf62c02e2363b0695e9c7e25a93449bd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"76653f7c556fac35b0a6b723d952b0de9fd9df80444614b88881532192459cf5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"c2fa2198346cf61422bd986d9ce744bde45180bf1e8b43388517312fcbacc025","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":3,"after_last_push_count":17,"trend_counts":[0,0,0,7,7,0,3],"trend_range":"2026-03-30..2026-04-05 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:132]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000dfab80\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-xhmz6z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-xhmz6z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0014f4090\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"3a55fa98-acad-4ad6-a240-181bf4b8ec64\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"ca924b49-6e46-45a9-868d-228ae0481199\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: 07C6F1DD1B8E413285DC76D57E2C9826 Ref B: CHI261140507060 Ref C: 2026-04-05T20:51:26Z\",\n ],\n \"Date\": [\n \"Sun, 05 Apr 2026 20:51:26 GMT\",\n ],\n \"Content-Length\": [\"107\"],\n \"X-Envoy-Upstream-Service-Time\": [\"302\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260405T205126Z:ca924b49-6e46-45a9-868d-228ae0481199\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Expires\": [\"-1\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/36803a68-b504-4c65-a064-df40a0d32fe8\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000dfaaa0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"management.azu...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:132]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000dfab80\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-xhmz6z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-xhmz6z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0014f4090\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Request-Id\": [\n \"3a55fa98-acad-4ad6-a240-181bf4b8ec64\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"ca924b49-6e46-45a9-868d-228ae0481199\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: 07C6F1DD1B8E413285DC76D57E2C9826 Ref B: CHI261140507060 Ref C: 2026-04-05T20:51:26Z\",\n ],\n \"Date\": [\n \"Sun, 05 Apr 2026 20:51:26 GMT\",\n ],\n \"Content-Length\": [\"107\"],\n \"X-Envoy-Upstream-Service-Time\": [\"302\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260405T205126Z:ca924b49-6e46-45a9-868d-228ae0481199\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Expires\": [\"-1\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/36803a68-b504-4c65-a064-df40a0d32fe8\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000dfaaa0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"management.azu...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:155]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000aa6120\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-j9zwrb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-j9zwrb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00093e060\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/6761ba3e-e3d5-4f2b-b121-1aaf18258695\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"X-Envoy-Upstream-Service-Time\": [\"158\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260405T205956Z:d0b57af8-bf86-42e3-81f6-8c47f1266249\",\n ],\n \"Date\": [\n \"Sun, 05 Apr 2026 20:59:56 GMT\",\n ],\n \"Content-Length\": [\"107\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 895D9C70FEA44594A28EC00C03902856 Ref B: CHI261140509062 Ref C: 2026-04-05T20:59:56Z\",\n ],\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"126ca117-9440-42c5-af31-856f795dd47d\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"d0b57af8-bf86-42e3-81f6-8c47f1266249\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000aa6060\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"m...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:155]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000aa6120\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-j9zwrb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-j9zwrb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/pullsecret-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00093e060\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/6761ba3e-e3d5-4f2b-b121-1aaf18258695\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"X-Envoy-Upstream-Service-Time\": [\"158\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260405T205956Z:d0b57af8-bf86-42e3-81f6-8c47f1266249\",\n ],\n \"Date\": [\n \"Sun, 05 Apr 2026 20:59:56 GMT\",\n ],\n \"Content-Length\": [\"107\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 895D9C70FEA44594A28EC00C03902856 Ref B: CHI261140509062 Ref C: 2026-04-05T20:59:56Z\",\n ],\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"126ca117-9440-42c5-af31-856f795dd47d\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"d0b57af8-bf86-42e3-81f6-8c47f1266249\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000aa6060\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"m...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:105]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00064c5a0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-d2x786/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-d2x786/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001695c50\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/37faf447-4cbb-45d8-ab1c-8f7f75feed59\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"1651\"],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"107\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260405T210827Z:1be05225-58f9-4e68-ae82-4585f065edaf\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Ms-Request-Id\": [\n \"f03a6005-7553-4eb6-9d49-9efe378f7a0f\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"1be05225-58f9-4e68-ae82-4585f065edaf\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: E4DC784654E84C74A2C16AB09A9371E1 Ref B: CHI261140508023 Ref C: 2026-04-05T21:08:25Z\",\n ],\n \"Date\": [\n \"Sun, 05 Apr 2026 21:08:26 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00064c380\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:105]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00064c5a0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-d2x786/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-d2x786/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/disabled-image-registry-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc001695c50\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/37faf447-4cbb-45d8-ab1c-8f7f75feed59\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"1651\"],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"107\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260405T210827Z:1be05225-58f9-4e68-ae82-4585f065edaf\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Ms-Request-Id\": [\n \"f03a6005-7553-4eb6-9d49-9efe378f7a0f\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"1be05225-58f9-4e68-ae82-4585f065edaf\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: E4DC784654E84C74A2C16AB09A9371E1 Ref B: CHI261140508023 Ref C: 2026-04-05T21:08:25Z\",\n ],\n \"Date\": [\n \"Sun, 05 Apr 2026 21:08:26 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00064c380\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040883810354073600","occurred_at":"2026-04-05T20:06:32Z","signature_id":"2d571bf651d0b8b4ee2843d8b226be03bddc79dec5efb9d4adef63867854ffa5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040883810354073600","occurred_at":"2026-04-05T20:06:32Z","signature_id":"38f51de79f43de40ad011679276e67513180cba797fb61bce2ce7134385db000","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040883810354073600","occurred_at":"2026-04-05T20:06:32Z","signature_id":"fd11db3f9cc963290c8c4f6ce44396e71a79bb480e4cfa44fbe51c0f78f68a27","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-30..2026-04-05 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-02","2026-03-16"],"prior_runs_affected":6,"prior_last_seen_at":"2026-03-18T21:39:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.004s.\nCluster should become ready within 45 minutes\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.004s.\nCluster should become ready within 45 minutes\nExpected\n \u003cbool\u003e: false\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"ee8099a3b414c84f0dabbe6ee3f745157c59a94f539cde4bae37fe8fe9711731","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout","failed_at":"unknown","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":7,"trend_counts":[0,0,0,2,2,0,1],"trend_range":"2026-03-30..2026-04-05 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-23"],"prior_runs_affected":7,"prior_last_seen_at":"2026-03-26T14:57:05Z","contributing_tests":[{"failed_at":"unknown","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":1}],"full_error_samples":["{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-05T17:06:11Z\"}"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040807616363368448","occurred_at":"2026-04-05T15:03:46Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"79d555cc6e5bd17f4b96af0c9e2da6a571d8241aacf20f84304c98773ef2682b","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"e2e-cilium-gpnkb5\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-30..2026-04-05 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-02"],"prior_runs_affected":3,"prior_last_seen_at":"2026-03-05T02:00:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:186]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0020085c0\u003e: \n failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"e2e-cilium-gpnkb5\" to finish creating: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"cilium-np\\\" for cluster \\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-gpnkb5\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:186]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0020085c0\u003e: \n failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"e2e-cilium-gpnkb5\" to finish creating: context deadline exceeded\n {\n msg: \"failed waiting for nodepool=\\\"cilium-np\\\" for cluster \\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-gpnkb5\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040610572545822720","occurred_at":"2026-04-05T02:00:47Z","signature_id":"2c3fed36a98d5c8fc924d06dac17540deb5718fccf37204fe17cb23c299f17c9","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}