{"meta":{"start_date":"2026-03-03","end_date":"2026-03-03","start_at":"2026-03-03T00:00:00Z","end_at":"2026-03-04T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T22:32:47Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":3,"failed_runs":3,"raw_occurrences":51,"matched_occurrences":51,"runs_affected":3},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-czs8jw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-czs8jw, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":21,"runs_affected":2,"failed_runs":2,"run_impact_percent":66.66666666666667,"anchor_occurrences":21,"after_last_push_count":21,"trend_counts":[0,0,0,0,0,0,21],"trend_range":"2026-02-25..2026-03-03 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010d49f0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-mhpzdj, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-mhpzdj\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-mhpzdj, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-mhpzdj, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-mhpzdj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-mhpzdj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f317f0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-mhpzdj\",\n },\n \u003c*fmt.wrapErrors | 0xc0010d49c0\u003e{\n msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-mhpzdj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-mhpzdj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f317f0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-mhpzdj\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010d49f0\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-mhpzdj, error: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-mhpzdj\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-mhpzdj, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster autoscaling-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-mhpzdj, error: failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-mhpzdj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-mhpzdj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f317f0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-mhpzdj\",\n },\n \u003c*fmt.wrapErrors | 0xc0010d49c0\u003e{\n msg: \"failed waiting for cluster=\\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-mhpzdj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-mhpzdj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f317f0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-mhpzdj\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000edcb40\u003e: \n failed to create HCP cluster ea-listgchttl, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listgchttl in resource group ea-list-knxn4r, error: failed waiting for cluster=\"ea-listgchttl\" in resourcegroup=\"ea-list-knxn4r\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listgchttl in resource group ea-list-knxn4r, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-listgchttl, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listgchttl in resource group ea-list-knxn4r, error: failed waiting for cluster=\\\"ea-listgchttl\\\" in resourcegroup=\\\"ea-list-knxn4r\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listgchttl in resource group ea-list-knxn4r, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012ca2b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listgchttl in resource group ea-list-knxn4r\",\n },\n \u003c*fmt.wrapErrors | 0xc000edcb10\u003e{\n msg: \"failed waiting for cluster=\\\"ea-listgchttl\\\" in resourcegroup=\\\"ea-list-knxn4r\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listgchttl in resource group ea-list-knxn4r, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012ca2b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listgchttl in resource group ea-list-knxn4r\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000edcb40\u003e: \n failed to create HCP cluster ea-listgchttl, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listgchttl in resource group ea-list-knxn4r, error: failed waiting for cluster=\"ea-listgchttl\" in resourcegroup=\"ea-list-knxn4r\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listgchttl in resource group ea-list-knxn4r, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-listgchttl, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listgchttl in resource group ea-list-knxn4r, error: failed waiting for cluster=\\\"ea-listgchttl\\\" in resourcegroup=\\\"ea-list-knxn4r\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listgchttl in resource group ea-list-knxn4r, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012ca2b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listgchttl in resource group ea-list-knxn4r\",\n },\n \u003c*fmt.wrapErrors | 0xc000edcb10\u003e{\n msg: \"failed waiting for cluster=\\\"ea-listgchttl\\\" in resourcegroup=\\\"ea-list-knxn4r\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listgchttl in resource group ea-list-knxn4r, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012ca2b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listgchttl in resource group ea-list-knxn4r\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:147]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f2f2f0\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-czs8jw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-czs8jw, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-czs8jw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-czs8jw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011f5510\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-czs8jw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:147]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f2f2f0\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-czs8jw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-czs8jw, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-czs8jw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-czs8jw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0011f5510\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-czs8jw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028977917446328320","occurred_at":"2026-03-03T23:36:46Z","signature_id":"017b698d7178d286945bcfe4d6e994b2c02aafa5970c9a52e61fda3c6d5e5d40","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028977917446328320","occurred_at":"2026-03-03T23:36:46Z","signature_id":"08e32b13d0d13b6f616ef4b926ae68e8c375a0b3eec441f5679337c17070e83a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028977917446328320","occurred_at":"2026-03-03T23:36:46Z","signature_id":"32e2d3e333f7bacf269ac406945ed1f41438d7fe4fdc1316d93ace5d8cda1d7f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028977917446328320","occurred_at":"2026-03-03T23:36:46Z","signature_id":"33ad34c8fd03669f0f19326c1670a357f83af9b0ccabaeaf49a11ddde575cfea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028977917446328320","occurred_at":"2026-03-03T23:36:46Z","signature_id":"48528b5d36836272263a61f0bdeb203d4adda8896dfc531c1262dd3b64f814f4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028977917446328320","occurred_at":"2026-03-03T23:36:46Z","signature_id":"4cbbe94d6a9f61a7e2f528c5405596e679d8b7e8c6e2b08c11213dfb3a586832","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028977917446328320","occurred_at":"2026-03-03T23:36:46Z","signature_id":"5dd24788e92e3a4e70b6c91655ec92f0b769b36d8bd6dd77d2e86b77cc19db70","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028977917446328320","occurred_at":"2026-03-03T23:36:46Z","signature_id":"8f0996009e219310eaadeff43d2356e8f383465b06edc49162c1e68d5e37b8b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028977917446328320","occurred_at":"2026-03-03T23:36:46Z","signature_id":"b7da27507704f520ee648f5b22760c5e1f157832003e8b9a576b7def8d12f8ed","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028977917446328320","occurred_at":"2026-03-03T23:36:46Z","signature_id":"da1649d9c01e52e74de987823bc31b8477079e3b385a270a7fc4fe6fe9b2cf4e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028977917446328320","occurred_at":"2026-03-03T23:36:46Z","signature_id":"e84a3edb0819d025ea656e556c245f15127971b9301ef5b29ff1c247b75bd638","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028961734651285504","occurred_at":"2026-03-03T22:32:27Z","signature_id":"0a1fe8b7a2dc791516446f2e9aa5c4671ef30fa1902d241516cd72a905bf2ded","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028961734651285504","occurred_at":"2026-03-03T22:32:27Z","signature_id":"0de8a23305e243ed316ae64f801eb973d905f168b7920f3de04c61e5dea16bbd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028961734651285504","occurred_at":"2026-03-03T22:32:27Z","signature_id":"243a092c9a85423cf1db68ae6a67c9d4b4dda0225953dac2f58025b89a788011","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028961734651285504","occurred_at":"2026-03-03T22:32:27Z","signature_id":"7ae73bcb2719e7bd99a0820ed464866061f12e140dd95a05da17a252dca8b0ef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028961734651285504","occurred_at":"2026-03-03T22:32:27Z","signature_id":"7c84e0d1efad91ef0c5dc4d45902ba15c7c522b9bb8b309194f77ea565adeae9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028961734651285504","occurred_at":"2026-03-03T22:32:27Z","signature_id":"908e57a2aeaf5f7680879396414d0255edabb476d02e90845ee1ebbdc84bde6b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028961734651285504","occurred_at":"2026-03-03T22:32:27Z","signature_id":"9ad50d27b27298e8c7156c5739d541207a36980aa4a0d667f5fa72683fb47d8b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028961734651285504","occurred_at":"2026-03-03T22:32:27Z","signature_id":"b42ef3c6c8b9574f2cf98e8d68f27a4b30e0278698c88f6b3ef48f8a7d613a26","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028961734651285504","occurred_at":"2026-03-03T22:32:27Z","signature_id":"ede8c9e136d435e0e3e588365f254232b874e5e973702d7d2802cf60a2f4b995","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028961734651285504","occurred_at":"2026-03-03T22:32:27Z","signature_id":"fbdd6877a93905611b743f99b66c9645b0d9e95f458b3edb182eb675fb166079","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":12,"runs_affected":2,"failed_runs":2,"run_impact_percent":66.66666666666667,"anchor_occurrences":12,"after_last_push_count":15,"trend_counts":[0,1,1,0,0,1,12],"trend_range":"2026-02-25..2026-03-03 UTC","prior_windows_present":1,"prior_window_starts":["2026-02-23"],"prior_runs_affected":5,"prior_last_seen_at":"2026-02-27T02:00:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:47]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:47]: Interrupted by User","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36]: Interrupted by User","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:30]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:30]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028977917446328320","occurred_at":"2026-03-03T23:36:46Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028977917446328320","occurred_at":"2026-03-03T23:36:46Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028977917446328320","occurred_at":"2026-03-03T23:36:46Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028977917446328320","occurred_at":"2026-03-03T23:36:46Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028977917446328320","occurred_at":"2026-03-03T23:36:46Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028961734651285504","occurred_at":"2026-03-03T22:32:27Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028961734651285504","occurred_at":"2026-03-03T22:32:27Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028961734651285504","occurred_at":"2026-03-03T22:32:27Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028961734651285504","occurred_at":"2026-03-03T22:32:27Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028961734651285504","occurred_at":"2026-03-03T22:32:27Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028961734651285504","occurred_at":"2026-03-03T22:32:27Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028961734651285504","occurred_at":"2026-03-03T22:32:27Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-kwvkmn\" in resourcegroup=\"admin-credential-lifecycle-test-h4bz6t\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-kwvkmn in resource group admin-credential-lifecycle-test-h4bz6t, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":6,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":6,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,0,6],"trend_range":"2026-02-25..2026-03-03 UTC","prior_windows_present":1,"prior_window_starts":["2026-02-23"],"prior_runs_affected":4,"prior_last_seen_at":"2026-03-01T02:00:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0007de9f0\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-kwvkmn\" in resourcegroup=\"admin-credential-lifecycle-test-h4bz6t\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-kwvkmn in resource group admin-credential-lifecycle-test-h4bz6t, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-kwvkmn\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-h4bz6t\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-kwvkmn in resource group admin-credential-lifecycle-test-h4bz6t, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a8b2e0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-kwvkmn in resource group admin-credential-lifecycle-test-h4bz6t\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0007de9f0\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-kwvkmn\" in resourcegroup=\"admin-credential-lifecycle-test-h4bz6t\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-kwvkmn in resource group admin-credential-lifecycle-test-h4bz6t, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-kwvkmn\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-h4bz6t\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-kwvkmn in resource group admin-credential-lifecycle-test-h4bz6t, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a8b2e0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-kwvkmn in resource group admin-credential-lifecycle-test-h4bz6t\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:100]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000d74b10\u003e: \n failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-jqxbnn\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-jqxbnn, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-jqxbnn\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-jqxbnn, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a12210\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-jqxbnn\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:100]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000d74b10\u003e: \n failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-jqxbnn\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-jqxbnn, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-jqxbnn\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-jqxbnn, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a12210\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-jqxbnn\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001488b40\u003e: \n failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-svgd9j\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-svgd9j, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-svgd9j\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-svgd9j, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0002ff5c0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-svgd9j\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001488b40\u003e: \n failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-svgd9j\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-svgd9j, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-svgd9j\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-svgd9j, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0002ff5c0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-svgd9j\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028651804983562240","occurred_at":"2026-03-03T02:00:55Z","signature_id":"2d647a85f21d23214b8650e69acc71e0976c96423f5fb6521469473f1bb2f867","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028651804983562240","occurred_at":"2026-03-03T02:00:55Z","signature_id":"701ddc4f9ef7aeffafaa7e009ebf57910eb68a7c9fd31115d3d90b133b4bfd52","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028651804983562240","occurred_at":"2026-03-03T02:00:55Z","signature_id":"7680ed401c09a5a8034ffa3dcb134a37e19bc575f66b0050dbffffb31b563b16","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028651804983562240","occurred_at":"2026-03-03T02:00:55Z","signature_id":"b5115187ac12066c81d4af01b9115d9641f2a7fc8faf5895603df500a2d38089","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028651804983562240","occurred_at":"2026-03-03T02:00:55Z","signature_id":"d095609085b62abd2a2bcb9901bc61ed3d828abca0489275343694139a551f83","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028651804983562240","occurred_at":"2026-03-03T02:00:55Z","signature_id":"e98054501a4f7933b4acd88af42376e57bbcc8ff2b128a0ccf07f15ac354198b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":5,"runs_affected":3,"failed_runs":3,"run_impact_percent":100,"anchor_occurrences":5,"after_last_push_count":13,"trend_counts":[0,0,2,3,2,1,5],"trend_range":"2026-02-25..2026-03-03 UTC","prior_windows_present":1,"prior_window_starts":["2026-02-23"],"prior_runs_affected":7,"prior_last_seen_at":"2026-03-01T02:00:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["eadLoop).run(0xc000396fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:2258 +0xca fp=0xc000396f60 sp=0xc000396e18 pc=0x31a62aa\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc001552380)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:2127 +0x52 fp=0xc000396fc8 sp=0xc000396f60 pc=0x31a5632\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:880 +0x25 fp=0xc000396fe0 sp=0xc000396fc8 pc=0x319edc5\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000396fe8 sp=0xc000396fe0 pc=0x2ac5921\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 137\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:880 +0xda5\n\ngoroutine 43 gp=0xc000002000 m=nil [runnable]:\nruntime.gopark(0xc0000b4f88?, 0x3?, 0x0?, 0x0?, 0xc0000b4f6a?)\n\t/usr/local/go/src/runtime/proc.go:460 +0xce fp=0xc0000b4df0 sp=0xc0000b4dd0 pc=0x2abd28e\nruntime.selectgo(0xc0000b4f88, 0xc0000b4f64, 0x0?, 0x0, 0x0?, 0x1)\n\t/usr/local/go/src/runtime/select.go:351 +0x8b7 fp=0xc0000b4f30 sp=0xc0000b4df0 pc=0x2a98137\ngithub.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2(0xc0000b4fa0?)\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:131 +0x7d fp=0xc0000b4fc8 sp=0xc0000b4f30 pc=0x2ed51fd\ngithub.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.gowrap1()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:158 +0x24 fp=0xc0000b4fe0 sp=0xc0000b4fc8 pc=0x2ed5144\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000b4fe8 sp=0xc0000b4fe0 pc=0x2ac5921\ncreated by github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts in goroutine 63\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:128 +0x165\n\ngoroutine 45 gp=0xc000e3a1c0 m=nil [select]:\nruntime.gopark(0xc0013d8fb0?, 0x2?, 0xe8?, 0x80?, 0xc0013d8fa4?)\n\t/usr/local/go/src/runtime/proc.go:460 +0xce fp=0xc0013d8e30 sp=0xc0013d8e10 pc=0x2abd28e\nruntime.selectgo(0xc0013d8fb0, 0xc0013d8fa0, 0xc0018146a0?, 0x0, 0x328a93c?, 0x1)\n\t/usr/local/go/src/runtime/select.go:351 +0x8b7 fp=0xc0013d8f70 sp=0xc0013d8e30 pc=0x2a98137\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x820bb58, 0xc0009d6640}, 0x77f1f7b?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36 +0x76 fp=0xc0013d8fe0 sp=0xc0013d8f70 pc=0x328a096\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x823e9a0, {0x820bb58, 0xc0009d6640}, 0x0)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284 +0x7b2 fp=0xc0013d9328 sp=0xc0013d8fe0 pc=0x371d0f2\ngithub.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x820bbc8?, 0xc000694380?}, {{0x82212e8?, 0xc0004e6400?}, 0xc001476b10?}, 0xc0013d9810, {0xc001418930, 0x16}, {0x780dcac, 0x12}, ...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:641 +0x405 fp=0xc0013d95a0 sp=0xc0013d9328 pc=0x4e941e5\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x77d2694, 0x4}, ...}, ...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:344 +0x497 fp=0xc0013d98e0 sp=0xc0013d95a0 pc=0x4e89d77\ngithub.com/Azure/ARO-HCP/test/e2e.init.func23.3({0x7fa3f060bc60, 0xc000b10330})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:160 +0x3a6 fp=0xc0013d9f58 sp=0xc0013d98e0 pc=0x501adc6\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213c0?, 0xc000b10330?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e fp=0xc0013d9f88 sp=0xc0013d9f58 pc=0x2eed25e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc0013d9fe0 sp=0xc0013d9f88 pc=0x2f0197b\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0013d9fe8 sp=0xc0013d9fe0 pc=0x2ac5921\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 63\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nrax 0xfffffffffffffffc\nrbx 0x5\nrcx 0x2a3ce0e\nrdx 0x80\nrdi 0x5\nrsi 0x7ffe2a7b78bc\nrbp 0x7ffe2a7b7870\nrsp 0x7ffe2a7b7828\nr8 0x0\nr9 0x0\nr10 0x2588\nr11 0x246\nr12 0x7ffe2a7b7900\nr13 0x1\nr14 0xb2ed360\nr15 0x7\nrip 0x2a3ce0e\nrflags 0x246\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\neadLoop).run(0xc000396fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:2258 +0xca fp=0xc000396f60 sp=0xc000396e18 pc=0x31a62aa\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc001552380)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:2127 +0x52 fp=0xc000396fc8 sp=0xc000396f60 pc=0x31a5632\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:880 +0x25 fp=0xc000396fe0 sp=0xc000396fc8 pc=0x319edc5\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000396fe8 sp=0xc000396fe0 pc=0x2ac5921\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 137\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:880 +0xda5\n\ngoroutine 43 gp=0xc000002000 m=nil [runnable]:\nruntime.gopark(0xc0000b4f88?, 0x3?, 0x0?, 0x0?, 0xc0000b4f6a?)\n\t/usr/local/go/src/runtime/proc.go:460 +0xce fp=0xc0000b4df0 sp=0xc0000b4dd0 pc=0x2abd28e\nruntime.selectgo(0xc0000b4f88, 0xc0000b4f64, 0x0?, 0x0, 0x0?, 0x1)\n\t/usr/local/go/src/runtime/select.go:351 +0x8b7 fp=0xc0000b4f30 sp=0xc0000b4df0 pc=0x2a98137\ngithub.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2(0xc0000b4fa0?)\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:131 +0x7d fp=0xc0000b4fc8 sp=0xc0000b4f30 pc=0x2ed51fd\ngithub.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.gowrap1()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:158 +0x24 fp=0xc0000b4fe0 sp=0xc0000b4fc8 pc=0x2ed5144\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000b4fe8 sp=0xc0000b4fe0 pc=0x2ac5921\ncreated by github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts in goroutine 63\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:128 +0x165\n\ngoroutine 45 gp=0xc000e3a1c0 m=nil [select]:\nruntime.gopark(0xc0013d8fb0?, 0x2?, 0xe8?, 0x80?, 0xc0013d8fa4?)\n\t/usr/local/go/src/runtime/proc.go:460 +0xce fp=0xc0013d8e30 sp=0xc0013d8e10 pc=0x2abd28e\nruntime.selectgo(0xc0013d8fb0, 0xc0013d8fa0, 0xc0018146a0?, 0x0, 0x328a93c?, 0x1)\n\t/usr/local/go/src/runtime/select.go:351 +0x8b7 fp=0xc0013d8f70 sp=0xc0013d8e30 pc=0x2a98137\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x820bb58, 0xc0009d6640}, 0x77f1f7b?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36 +0x76 fp=0xc0013d8fe0 sp=0xc0013d8f70 pc=0x328a096\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x823e9a0, {0x820bb58, 0xc0009d6640}, 0x0)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284 +0x7b2 fp=0xc0013d9328 sp=0xc0013d8fe0 pc=0x371d0f2\ngithub.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x820bbc8?, 0xc000694380?}, {{0x82212e8?, 0xc0004e6400?}, 0xc001476b10?}, 0xc0013d9810, {0xc001418930, 0x16}, {0x780dcac, 0x12}, ...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:641 +0x405 fp=0xc0013d95a0 sp=0xc0013d9328 pc=0x4e941e5\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x77d2694, 0x4}, ...}, ...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:344 +0x497 fp=0xc0013d98e0 sp=0xc0013d95a0 pc=0x4e89d77\ngithub.com/Azure/ARO-HCP/test/e2e.init.func23.3({0x7fa3f060bc60, 0xc000b10330})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:160 +0x3a6 fp=0xc0013d9f58 sp=0xc0013d98e0 pc=0x501adc6\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213c0?, 0xc000b10330?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e fp=0xc0013d9f88 sp=0xc0013d9f58 pc=0x2eed25e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc0013d9fe0 sp=0xc0013d9f88 pc=0x2f0197b\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0013d9fe8 sp=0xc0013d9fe0 pc=0x2ac5921\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 63\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nrax 0xfffffffffffffffc\nrbx 0x5\nrcx 0x2a3ce0e\nrdx 0x80\nrdi 0x5\nrsi 0x7ffe2a7b78bc\nrbp 0x7ffe2a7b7870\nrsp 0x7ffe2a7b7828\nr8 0x0\nr9 0x0\nr10 0x2588\nr11 0x246\nr12 0x7ffe2a7b7900\nr13 0x1\nr14 0xb2ed360\nr15 0x7\nrip 0x2a3ce0e\nrflags 0x246\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command","\u003e github.com/Azure/ARO-HCP/test/e2e.init.func16.2({0x7feaf8330288, 0xc0004da8d0})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n | \n | By(\"waiting for first cluster to complete creation\")\n \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n | \tFrequency: framework.StandardPollInterval,\n | })\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213c0?, 0xc0004da8d0?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 101\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/04/26 01:16:11.849\n \"ts\"=\"2026-03-04 01:16:11.849474\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-04 01:16:11.849578\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-z9pkgz\" \"resourceGroup\"=\"customer-rg-vhsgnq\"\n \"ts\"=\"2026-03-04 01:16:12.326078\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-z9pkgz\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-04 01:16:13.573758\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-tz2b4w\" \"resourceGroup\"=\"customer-rg-vhsgnq\"\n \"ts\"=\"2026-03-04 01:16:13.680973\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-tz2b4w\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-04 01:16:15.111085\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-04 01:16:15.111218\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-04 01:16:15.111311\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-04 01:16:15.112165\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-105\"\n \"ts\"=\"2026-03-04 01:16:15.112207\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-87\"\n \"ts\"=\"2026-03-04 01:16:15.112208\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-vhsgnq\"\n \"ts\"=\"2026-03-04 01:16:15.112244\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-43\"\n \"ts\"=\"2026-03-04 01:16:15.270517\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-vhsgnq\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n \"ts\"=\"2026-03-04 01:16:15.273588\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-87\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-04 01:16:15.341558\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-vhsgnq\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-tz2b4w\"\n \"ts\"=\"2026-03-04 01:16:15.424275\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-43\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-04 01:16:15.495136\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-vhsgnq\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n \"ts\"=\"2026-03-04 01:16:15.553286\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-43\" \"deployment\"=\"msi-deployment-43\"\n \"ts\"=\"2026-03-04 01:16:15.570032\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-vhsgnq\" \"deployment\"=\"customer-infra-basic-hcp-cluster-z9pkgz\"\n \"ts\"=\"2026-03-04 01:16:15.813903\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-105\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-04 01:16:15.832680\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-87\" \"deployment\"=\"msi-deployment-87\"\n \"ts\"=\"2026-03-04 01:16:16.124957\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-105\" \"deployment\"=\"msi-deployment-105\"\n \"ts\"=\"2026-03-04 01:16:16.281359\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-04 01:16:16.281545\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-04 01:16:16.281745\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-vhsgnq\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n\u003e github.com/Azure/ARO-HCP/test/e2e.init.func16.2({0x7feaf8330288, 0xc0004da8d0})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n | \n | By(\"waiting for first cluster to complete creation\")\n \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n | \tFrequency: framework.StandardPollInterval,\n | })\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213c0?, 0xc0004da8d0?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 101\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/04/26 01:16:11.849\n \"ts\"=\"2026-03-04 01:16:11.849474\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-04 01:16:11.849578\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-z9pkgz\" \"resourceGroup\"=\"customer-rg-vhsgnq\"\n \"ts\"=\"2026-03-04 01:16:12.326078\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-z9pkgz\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-04 01:16:13.573758\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-tz2b4w\" \"resourceGroup\"=\"customer-rg-vhsgnq\"\n \"ts\"=\"2026-03-04 01:16:13.680973\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-tz2b4w\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-04 01:16:15.111085\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-04 01:16:15.111218\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-04 01:16:15.111311\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-04 01:16:15.112165\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-105\"\n \"ts\"=\"2026-03-04 01:16:15.112207\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-87\"\n \"ts\"=\"2026-03-04 01:16:15.112208\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-vhsgnq\"\n \"ts\"=\"2026-03-04 01:16:15.112244\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-43\"\n \"ts\"=\"2026-03-04 01:16:15.270517\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-vhsgnq\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n \"ts\"=\"2026-03-04 01:16:15.273588\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-87\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-04 01:16:15.341558\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-vhsgnq\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-tz2b4w\"\n \"ts\"=\"2026-03-04 01:16:15.424275\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-43\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-04 01:16:15.495136\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-vhsgnq\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n \"ts\"=\"2026-03-04 01:16:15.553286\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-43\" \"deployment\"=\"msi-deployment-43\"\n \"ts\"=\"2026-03-04 01:16:15.570032\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-vhsgnq\" \"deployment\"=\"customer-infra-basic-hcp-cluster-z9pkgz\"\n \"ts\"=\"2026-03-04 01:16:15.813903\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-105\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-04 01:16:15.832680\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-87\" \"deployment\"=\"msi-deployment-87\"\n \"ts\"=\"2026-03-04 01:16:16.124957\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-105\" \"deployment\"=\"msi-deployment-105\"\n \"ts\"=\"2026-03-04 01:16:16.281359\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-04 01:16:16.281545\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-04 01:16:16.281745\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-vhsgnq\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","\u003e github.com/Azure/ARO-HCP/test/e2e.init.func16.2({0x7f6140169e60, 0xc00031c750})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n | \n | By(\"waiting for first cluster to complete creation\")\n \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n | \tFrequency: framework.StandardPollInterval,\n | })\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213c0?, 0xc00031c750?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 95\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/04/26 00:05:02.077\n \"ts\"=\"2026-03-04 00:05:02.077648\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-04 00:05:02.077755\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-8mgz7c\" \"resourceGroup\"=\"customer-rg-gxwvdd\"\n \"ts\"=\"2026-03-04 00:05:02.493858\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-8mgz7c\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-04 00:05:03.895887\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-x9tzjh\" \"resourceGroup\"=\"customer-rg-gxwvdd\"\n \"ts\"=\"2026-03-04 00:05:03.991400\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-x9tzjh\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-04 00:05:05.282872\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-04 00:05:05.282983\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-04 00:05:05.283067\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-04 00:05:05.283817\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-90\"\n \"ts\"=\"2026-03-04 00:05:05.283830\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-123\"\n \"ts\"=\"2026-03-04 00:05:05.283852\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-144\"\n \"ts\"=\"2026-03-04 00:05:05.283906\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-gxwvdd\"\n \"ts\"=\"2026-03-04 00:05:05.467033\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-gxwvdd\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n \"ts\"=\"2026-03-04 00:05:05.530440\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-gxwvdd\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-x9tzjh\"\n \"ts\"=\"2026-03-04 00:05:05.607887\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-144\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-04 00:05:05.617180\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-gxwvdd\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n \"ts\"=\"2026-03-04 00:05:05.687502\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-gxwvdd\" \"deployment\"=\"customer-infra-basic-hcp-cluster-8mgz7c\"\n \"ts\"=\"2026-03-04 00:05:05.770906\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-144\" \"deployment\"=\"msi-deployment-144\"\n \"ts\"=\"2026-03-04 00:05:05.791163\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-123\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-04 00:05:05.920959\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-123\" \"deployment\"=\"msi-deployment-123\"\n \"ts\"=\"2026-03-04 00:05:05.982538\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-90\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-04 00:05:06.722919\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-90\" \"deployment\"=\"msi-deployment-90\"\n \"ts\"=\"2026-03-04 00:05:06.881204\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-04 00:05:06.881353\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-04 00:05:06.881480\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-gxwvdd\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n\u003e github.com/Azure/ARO-HCP/test/e2e.init.func16.2({0x7f6140169e60, 0xc00031c750})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n | \n | By(\"waiting for first cluster to complete creation\")\n \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n | \tFrequency: framework.StandardPollInterval,\n | })\n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213c0?, 0xc00031c750?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 95\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/04/26 00:05:02.077\n \"ts\"=\"2026-03-04 00:05:02.077648\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-03-04 00:05:02.077755\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-8mgz7c\" \"resourceGroup\"=\"customer-rg-gxwvdd\"\n \"ts\"=\"2026-03-04 00:05:02.493858\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-8mgz7c\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-04 00:05:03.895887\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-x9tzjh\" \"resourceGroup\"=\"customer-rg-gxwvdd\"\n \"ts\"=\"2026-03-04 00:05:03.991400\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-x9tzjh\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-03-04 00:05:05.282872\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-04 00:05:05.282983\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n \"ts\"=\"2026-03-04 00:05:05.283067\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-03-04 00:05:05.283817\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-90\"\n \"ts\"=\"2026-03-04 00:05:05.283830\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-123\"\n \"ts\"=\"2026-03-04 00:05:05.283852\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-144\"\n \"ts\"=\"2026-03-04 00:05:05.283906\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-gxwvdd\"\n \"ts\"=\"2026-03-04 00:05:05.467033\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-gxwvdd\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n \"ts\"=\"2026-03-04 00:05:05.530440\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-gxwvdd\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-x9tzjh\"\n \"ts\"=\"2026-03-04 00:05:05.607887\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-144\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-04 00:05:05.617180\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-gxwvdd\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n \"ts\"=\"2026-03-04 00:05:05.687502\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-gxwvdd\" \"deployment\"=\"customer-infra-basic-hcp-cluster-8mgz7c\"\n \"ts\"=\"2026-03-04 00:05:05.770906\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-144\" \"deployment\"=\"msi-deployment-144\"\n \"ts\"=\"2026-03-04 00:05:05.791163\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-123\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-04 00:05:05.920959\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-123\" \"deployment\"=\"msi-deployment-123\"\n \"ts\"=\"2026-03-04 00:05:05.982538\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-90\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-03-04 00:05:06.722919\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-90\" \"deployment\"=\"msi-deployment-90\"\n \"ts\"=\"2026-03-04 00:05:06.881204\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-03-04 00:05:06.881353\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-03-04 00:05:06.881480\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-gxwvdd\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028977917446328320","occurred_at":"2026-03-03T23:36:46Z","signature_id":"369dd9382f8d7b890587a7a1bdc0940fac0c81cf54a56b113066964c543b6c94","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028977917446328320","occurred_at":"2026-03-03T23:36:46Z","signature_id":"70a7cae78b510f8f18bdd37dcc8aba81435a3eb5622197cbe895aadf1b90c932","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028961734651285504","occurred_at":"2026-03-03T22:32:27Z","signature_id":"bda0a0e6979d2390f65e8fca864a0635a7ea4ef1201455a621f24f1c69b1e55a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028651804983562240","occurred_at":"2026-03-03T02:00:55Z","signature_id":"2a68f81be769ce011036801b5d26494ccbbf700474745e3e1fe3b49936d95b9d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028651804983562240","occurred_at":"2026-03-03T02:00:55Z","signature_id":"cef96c2a79670ec90e6097efdbc7e1d97d7c729507cc78a7c4a5f743964a8fda","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-82wj5d, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-82wj5d\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-02-25..2026-03-03 UTC","prior_windows_present":1,"prior_window_starts":["2026-02-23"],"prior_runs_affected":6,"prior_last_seen_at":"2026-03-01T02:00:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001370810\u003e: \n failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-45vbz8, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-45vbz8\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-45vbz8, error: failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-45vbz8\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00070b5d0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-45vbz8\",\n },\n \u003c*fmt.wrapError | 0xc0009d0520\u003e{\n msg: \"failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-45vbz8\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001370810\u003e: \n failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-45vbz8, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-45vbz8\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-45vbz8, error: failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-45vbz8\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00070b5d0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-45vbz8\",\n },\n \u003c*fmt.wrapError | 0xc0009d0520\u003e{\n msg: \"failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-45vbz8\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:124]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015a2510\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-2x577m, error: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-2x577m\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-2x577m, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-2x577m\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000893b80\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-2x577m\",\n },\n \u003c*fmt.wrapError | 0xc001406360\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-2x577m\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:124]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015a2510\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-2x577m, error: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-2x577m\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-2x577m, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-2x577m\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000893b80\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-2x577m\",\n },\n \u003c*fmt.wrapError | 0xc001406360\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-2x577m\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001484810\u003e: \n failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-82wj5d, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-82wj5d\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-82wj5d, error: failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-82wj5d\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012cb2a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-82wj5d\",\n },\n \u003c*fmt.wrapError | 0xc00091c5c0\u003e{\n msg: \"failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-82wj5d\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001484810\u003e: \n failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-82wj5d, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-82wj5d\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-82wj5d, error: failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-82wj5d\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012cb2a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-82wj5d\",\n },\n \u003c*fmt.wrapError | 0xc00091c5c0\u003e{\n msg: \"failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-82wj5d\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028651804983562240","occurred_at":"2026-03-03T02:00:55Z","signature_id":"249b75096d06dfbb9edb0a1ad56e1beee069fd484b55fc96173f3e4dd807bfd6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028651804983562240","occurred_at":"2026-03-03T02:00:55Z","signature_id":"2c748cbb4ada12c0e6f8e046cc4f95d857c0b3ec3eaa866c389d86e8a26deacb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028651804983562240","occurred_at":"2026-03-03T02:00:55Z","signature_id":"63f43c3557360dead2bbb5864e395706223a3c2479e7266d5240f2302f4902bb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028651804983562240","occurred_at":"2026-03-03T02:00:55Z","signature_id":"e088e678dfd9e62ed0375406977e6fea81ee5378f115e9ed890ac7f3939a7367","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":66.66666666666667,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-02-25..2026-03-03 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.145s.\nCluster should become ready within 45 minutes\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.145s.\nCluster should become ready within 45 minutes\nExpected\n \u003cbool\u003e: false\nto be true","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.006s.\nCluster should become ready within 45 minutes\nExpected\n \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.006s.\nCluster should become ready within 45 minutes\nExpected\n \u003cbool\u003e: false\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028977917446328320","occurred_at":"2026-03-03T23:36:46Z","signature_id":"820619bcd5687b3bfe243ccc47aa109fc20b546f9fe3986b85846fabfd3d1e59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028961734651285504","occurred_at":"2026-03-03T22:32:27Z","signature_id":"4f3fda9076e5a0e453dc8330472cdfbcd9f2b41ce57edf1d1a6a79a6f3910ba0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f1386f529db6e94f57caba5baeea25a7ed546b620757afe7e7438f6ec564ee32","failure_pattern":"CreateOrUpdateExternalAuthAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for external auth \"ea-list\" in resourcegroup=\"ea-list-ls8jkz\" for cluster=\"ea-listdwzhrk\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-listdwzhrk in resource group ea-list-ls8jkz, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-02-25..2026-03-03 UTC","prior_windows_present":1,"prior_window_starts":["2026-02-23"],"prior_runs_affected":6,"prior_last_seen_at":"2026-03-01T02:00:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:118]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000eccd20\u003e: \n failed waiting for external auth \"ea-list\" in resourcegroup=\"ea-list-ls8jkz\" for cluster=\"ea-listdwzhrk\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-listdwzhrk in resource group ea-list-ls8jkz, error: context deadline exceeded\n {\n msg: \"failed waiting for external auth \\\"ea-list\\\" in resourcegroup=\\\"ea-list-ls8jkz\\\" for cluster=\\\"ea-listdwzhrk\\\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-listdwzhrk in resource group ea-list-ls8jkz, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00083abf0\u003e{\n s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-listdwzhrk in resource group ea-list-ls8jkz\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:118]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000eccd20\u003e: \n failed waiting for external auth \"ea-list\" in resourcegroup=\"ea-list-ls8jkz\" for cluster=\"ea-listdwzhrk\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-listdwzhrk in resource group ea-list-ls8jkz, error: context deadline exceeded\n {\n msg: \"failed waiting for external auth \\\"ea-list\\\" in resourcegroup=\\\"ea-list-ls8jkz\\\" for cluster=\\\"ea-listdwzhrk\\\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-listdwzhrk in resource group ea-list-ls8jkz, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00083abf0\u003e{\n s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-listdwzhrk in resource group ea-list-ls8jkz\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2028651804983562240","occurred_at":"2026-03-03T02:00:55Z","signature_id":"5d120e3b14905958df7cb9d7acc162ae843b19f8a37edd16d244139511c4f11e","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}