{"meta":{"start_date":"2026-03-07","end_date":"2026-03-07","start_at":"2026-03-07T00:00:00Z","end_at":"2026-03-08T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T21:36:19Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":6,"failed_runs":2,"raw_occurrences":22,"matched_occurrences":22,"runs_affected":2},"rows":[{"environment":"int","failure_pattern_id":"0ece011306dab74cd22dd19f1f3c89bbdb930a2321b61be59e4a6a90aae6e3eb","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-hrmh9p\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-hrmh9p, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":16,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":16,"after_last_push_count":16,"trend_counts":[0,0,0,0,0,0,16],"trend_range":"2026-03-01..2026-03-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-02-23"],"prior_runs_affected":9,"prior_last_seen_at":"2026-03-01T04:25:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00114ab40\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7hzpkh, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-7hzpkh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7hzpkh, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7hzpkh, error: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-7hzpkh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7hzpkh, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000b39de0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7hzpkh\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00114ab10\u003e{\n                msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-7hzpkh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7hzpkh, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000b39de0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7hzpkh\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00114ab40\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7hzpkh, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-7hzpkh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7hzpkh, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7hzpkh, error: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-7hzpkh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7hzpkh, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000b39de0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7hzpkh\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00114ab10\u003e{\n                msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-7hzpkh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7hzpkh, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000b39de0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-7hzpkh\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000b2df20\u003e: \n    failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-h6h8pw, error: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-h6h8pw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-h6h8pw, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-h6h8pw, error: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-h6h8pw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-h6h8pw, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00107ce50\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-h6h8pw\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000b2def0\u003e{\n                msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-h6h8pw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-h6h8pw, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc00107ce50\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-h6h8pw\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000b2df20\u003e: \n    failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-h6h8pw, error: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-h6h8pw\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-h6h8pw, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-h6h8pw, error: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-h6h8pw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-h6h8pw, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00107ce50\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-h6h8pw\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000b2def0\u003e{\n                msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-h6h8pw\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-h6h8pw, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc00107ce50\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-h6h8pw\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00154ee10\u003e: \n    failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-wstmzs, error: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-wstmzs\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-wstmzs, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-wstmzs, error: failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-wstmzs\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-wstmzs, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001478e50\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-wstmzs\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00154ede0\u003e{\n                msg: \"failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-wstmzs\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-wstmzs, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001478e50\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-wstmzs\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00154ee10\u003e: \n    failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-wstmzs, error: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-wstmzs\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-wstmzs, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-wstmzs, error: failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-wstmzs\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-wstmzs, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001478e50\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-wstmzs\",\n            },\n            \u003c*fmt.wrapErrors | 0xc00154ede0\u003e{\n                msg: \"failed waiting for cluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-wstmzs\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-wstmzs, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc001478e50\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-wstmzs\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2030395131827326976","occurred_at":"2026-03-07T21:28:16Z","signature_id":"072e841fd7f1a370e09707e073f2fda6b10e207dcc6b382f925aad85b185b50c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2030395131827326976","occurred_at":"2026-03-07T21:28:16Z","signature_id":"09e7e7aef6dda6df6fc7e7ced8d7ce0b83ea88aebba6d47cec763ba9cbdd69b2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2030395131827326976","occurred_at":"2026-03-07T21:28:16Z","signature_id":"0e19dab8d51dd8088e76e6ee5f2993c383bb2633b52ff6a2a5513e6ec22d1ea2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2030395131827326976","occurred_at":"2026-03-07T21:28:16Z","signature_id":"0e4633019277fc70d889fa25b3a49261e4a6ae836adc965465dd5c6e6ec467a8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2030395131827326976","occurred_at":"2026-03-07T21:28:16Z","signature_id":"280d026a012e8c48acd50fdcc5c3675b1f4b9b2abd71eea417d59fc13159a182","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2030395131827326976","occurred_at":"2026-03-07T21:28:16Z","signature_id":"302b0b444fe540e243f224773454745b265af5d4309c5f3230c673c9a94279f4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2030395131827326976","occurred_at":"2026-03-07T21:28:16Z","signature_id":"6582ee2efd71b881b17746923df30ad6e8977778c1cb7ef9253726cc575768bb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2030395131827326976","occurred_at":"2026-03-07T21:28:16Z","signature_id":"759bbb103a5aba1c356772d382313cc6ee8ae593d526a3f2acc6b9d096c2b7fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2030395131827326976","occurred_at":"2026-03-07T21:28:16Z","signature_id":"80f81a05c9012fc0f13e83741dd93bd8993d34af56663916facc46b7d261b8a8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2030395131827326976","occurred_at":"2026-03-07T21:28:16Z","signature_id":"829a37261817ed7f13b84d9a1ff15507c5732ef6941a076e8befbf6717493904","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2030395131827326976","occurred_at":"2026-03-07T21:28:16Z","signature_id":"85cb2841d618dbfa8efb97b767a5a0f0184acf849017e0f6ff1fb51f15e621c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2030395131827326976","occurred_at":"2026-03-07T21:28:16Z","signature_id":"8e95bdea8d670a50175656cdf09c9a64c45b4b58718100f27688b03c13e8c8a7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2030395131827326976","occurred_at":"2026-03-07T21:28:16Z","signature_id":"a048ba9e565c2c46407476d27c13f4aac8d9bb8dedbdce1907dc386f995de450","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2030395131827326976","occurred_at":"2026-03-07T21:28:16Z","signature_id":"c964283928876f9d77e3191c414666813b38459b74a0571feb56e9866129925c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2030395131827326976","occurred_at":"2026-03-07T21:28:16Z","signature_id":"da5a59ce0d7ec822c5d449bebadcbdfa29bb119cbb76234ee4be8e7b2e025721","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2030395131827326976","occurred_at":"2026-03-07T21:28:16Z","signature_id":"ea04a50587d9781ef21dfdd463ce8a577691138cb9083611f6befa75e53912d4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"d0993ee7c174e66d21d6873cdcbb83cae1352c1482345816cea0f68947670c76","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-03-01..2026-03-07 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1}],"full_error_samples":["urrent progress report:\n      Customer should respect cluster-wide node limits with nodepool autoscaling (Spec Runtime: 1h29m59.84s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:122\n        In [It] (Node Runtime: 1h29m59.84s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:122\n          At [By Step] creating cluster with MaxNodesTotal limit (Step Runtime: 23m49.32s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:143\n\n          Spec Goroutine\n          goroutine 44 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8214098, 0xc000734550}, 0x77f8912?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8246e80, {0x8214098, 0xc000734550}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8214108?, 0xc00036f340?}, {{0x8229828?, 0xc00082a180?}, 0xc000af0970?}, 0xc000095810, {0xc00138a2d0, 0x16}, {0x7814759, 0x12}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:641\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x77d9018, 0x4}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:344\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func22.3({0x7fd3a6726b10, 0xc0009509f0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:160\n                | Expect(err).NotTo(HaveOccurred())\n                | \n                \u003e err = tc.CreateHCPClusterFromParam(ctx,\n                | \tGinkgoLogr,\n                | \t*resourceGroup.Name,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8229900?, 0xc0009509f0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 107\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166 @ 03/07/26 23:09:29.214\n  \"ts\"=\"2026-03-07 23:09:29.214402\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-07 23:09:29.214455\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-node-limit-cluster-564z9f\" \"resourceGroup\"=\"node-limit-test-jsd8j7\"\n  \"ts\"=\"2026-03-07 23:09:29.539378\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-node-limit-cluster-564z9f\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-07 23:09:30.645108\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-133f4b3b85cd881bd2d7025c4df814bf698b577d4462803137df4f5ac0cf68e9.yaml.gz\"\n  \"ts\"=\"2026-03-07 23:09:30.645242\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-133f4b3b85cd881bd2d7025c4df814bf698b577d4462803137df4f5ac0cf68e9.yaml.gz\"\n  \"ts\"=\"2026-03-07 23:09:30.645309\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-07 23:09:30.646318\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-67\"\n  \"ts\"=\"2026-03-07 23:09:30.646334\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"node-limit-test-jsd8j7\"\n  \"ts\"=\"2026-03-07 23:09:30.808861\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-67\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-07 23:09:30.828467\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"node-limit-test-jsd8j7\" \"deployment\"=\"pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\"\n  \"ts\"=\"2026-03-07 23:09:30.959675\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"node-limit-test-jsd8j7\" \"deployment\"=\"customer-infra-node-limit-cluster-564z9f\"\n  \"ts\"=\"2026-03-07 23:09:31.116478\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-67\" \"deployment\"=\"msi-deployment-67\"\n  \"ts\"=\"2026-03-07 23:09:31.255318\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-07 23:09:31.255416\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-07 23:09:31.255513\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"node-limit-test-jsd8j7\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nurrent progress report:\n      Customer should respect cluster-wide node limits with nodepool autoscaling (Spec Runtime: 1h29m59.84s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:122\n        In [It] (Node Runtime: 1h29m59.84s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:122\n          At [By Step] creating cluster with MaxNodesTotal limit (Step Runtime: 23m49.32s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:143\n\n          Spec Goroutine\n          goroutine 44 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8214098, 0xc000734550}, 0x77f8912?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8246e80, {0x8214098, 0xc000734550}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8214108?, 0xc00036f340?}, {{0x8229828?, 0xc00082a180?}, 0xc000af0970?}, 0xc000095810, {0xc00138a2d0, 0x16}, {0x7814759, 0x12}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:641\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x77d9018, 0x4}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:344\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func22.3({0x7fd3a6726b10, 0xc0009509f0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:160\n                | Expect(err).NotTo(HaveOccurred())\n                | \n                \u003e err = tc.CreateHCPClusterFromParam(ctx,\n                | \tGinkgoLogr,\n                | \t*resourceGroup.Name,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8229900?, 0xc0009509f0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 107\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166 @ 03/07/26 23:09:29.214\n  \"ts\"=\"2026-03-07 23:09:29.214402\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-07 23:09:29.214455\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-node-limit-cluster-564z9f\" \"resourceGroup\"=\"node-limit-test-jsd8j7\"\n  \"ts\"=\"2026-03-07 23:09:29.539378\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-node-limit-cluster-564z9f\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-07 23:09:30.645108\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-133f4b3b85cd881bd2d7025c4df814bf698b577d4462803137df4f5ac0cf68e9.yaml.gz\"\n  \"ts\"=\"2026-03-07 23:09:30.645242\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-133f4b3b85cd881bd2d7025c4df814bf698b577d4462803137df4f5ac0cf68e9.yaml.gz\"\n  \"ts\"=\"2026-03-07 23:09:30.645309\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-07 23:09:30.646318\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-67\"\n  \"ts\"=\"2026-03-07 23:09:30.646334\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"node-limit-test-jsd8j7\"\n  \"ts\"=\"2026-03-07 23:09:30.808861\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-67\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-07 23:09:30.828467\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"node-limit-test-jsd8j7\" \"deployment\"=\"pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\"\n  \"ts\"=\"2026-03-07 23:09:30.959675\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"node-limit-test-jsd8j7\" \"deployment\"=\"customer-infra-node-limit-cluster-564z9f\"\n  \"ts\"=\"2026-03-07 23:09:31.116478\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-67\" \"deployment\"=\"msi-deployment-67\"\n  \"ts\"=\"2026-03-07 23:09:31.255318\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-07 23:09:31.255416\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-07 23:09:31.255513\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"node-limit-test-jsd8j7\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","p all remaining specs.  Interrupt again to skip cleanup.\n    Here's a current progress report:\n      Customer should be able to create node pools with ARM64-based VMs (Spec Runtime: 1h29m59.838s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:35\n        In [It] (Node Runtime: 1h29m59.838s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:35\n          At [By Step] creating the HCP cluster (Step Runtime: 22m55.63s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:82\n\n          Spec Goroutine\n          goroutine 105 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8214098, 0xc000a9e320}, 0x77f8912?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8246e80, {0x8214098, 0xc000a9e320}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8214108?, 0xc0006a04d0?}, {{0x8229828?, 0xc000755400?}, 0xc0008a8ae0?}, 0xc0000957e8, {0xc001440378, 0x17}, {0x781ed63, 0x14}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:641\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x77d9018, 0x4}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:344\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func4.2({0x7fcb20176e60, 0xc000c88780})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:83\n                | \n                | By(\"creating the HCP cluster\")\n                \u003e err = tc.CreateHCPClusterFromParam(\n                | \tctx,\n                | \tGinkgoLogr,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8229900?, 0xc000c88780?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 102\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90 @ 03/07/26 23:09:29.205\n  \"ts\"=\"2026-03-07 23:09:29.206087\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-07 23:09:29.206188\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-arm64-vm-hcp-cluster-qqhxlg\" \"resourceGroup\"=\"arm64-vm-cluster-b4f8bt\"\n  \"ts\"=\"2026-03-07 23:09:29.413482\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-arm64-vm-hcp-cluster-qqhxlg\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-07 23:09:31.163451\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-a6eb9d68aa9cc22f7e78141cf691002056d2ca6a7d34a8fba7d067b26fef9e86.yaml.gz\"\n  \"ts\"=\"2026-03-07 23:09:31.163568\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-a6eb9d68aa9cc22f7e78141cf691002056d2ca6a7d34a8fba7d067b26fef9e86.yaml.gz\"\n  \"ts\"=\"2026-03-07 23:09:31.163634\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-07 23:09:31.164640\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-10\"\n  \"ts\"=\"2026-03-07 23:09:31.164656\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"arm64-vm-cluster-b4f8bt\"\n  \"ts\"=\"2026-03-07 23:09:31.407942\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-10\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-07 23:09:31.474217\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"arm64-vm-cluster-b4f8bt\" \"deployment\"=\"pooledNonMsiScopedAssignments-6htyoibevj7ye\"\n  \"ts\"=\"2026-03-07 23:09:31.546551\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-10\" \"deployment\"=\"msi-deployment-10\"\n  \"ts\"=\"2026-03-07 23:09:31.617334\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"arm64-vm-cluster-b4f8bt\" \"deployment\"=\"customer-infra-arm64-vm-hcp-cluster-qqhxlg\"\n  \"ts\"=\"2026-03-07 23:09:31.781469\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-07 23:09:31.781570\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-07 23:09:31.781667\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"arm64-vm-cluster-b4f8bt\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\np all remaining specs.  Interrupt again to skip cleanup.\n    Here's a current progress report:\n      Customer should be able to create node pools with ARM64-based VMs (Spec Runtime: 1h29m59.838s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:35\n        In [It] (Node Runtime: 1h29m59.838s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:35\n          At [By Step] creating the HCP cluster (Step Runtime: 22m55.63s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:82\n\n          Spec Goroutine\n          goroutine 105 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8214098, 0xc000a9e320}, 0x77f8912?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8246e80, {0x8214098, 0xc000a9e320}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8214108?, 0xc0006a04d0?}, {{0x8229828?, 0xc000755400?}, 0xc0008a8ae0?}, 0xc0000957e8, {0xc001440378, 0x17}, {0x781ed63, 0x14}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:641\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0x77d9018, 0x4}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:344\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func4.2({0x7fcb20176e60, 0xc000c88780})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:83\n                | \n                | By(\"creating the HCP cluster\")\n                \u003e err = tc.CreateHCPClusterFromParam(\n                | \tctx,\n                | \tGinkgoLogr,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8229900?, 0xc000c88780?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 102\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90 @ 03/07/26 23:09:29.205\n  \"ts\"=\"2026-03-07 23:09:29.206087\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-07 23:09:29.206188\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-arm64-vm-hcp-cluster-qqhxlg\" \"resourceGroup\"=\"arm64-vm-cluster-b4f8bt\"\n  \"ts\"=\"2026-03-07 23:09:29.413482\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-arm64-vm-hcp-cluster-qqhxlg\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-07 23:09:31.163451\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-a6eb9d68aa9cc22f7e78141cf691002056d2ca6a7d34a8fba7d067b26fef9e86.yaml.gz\"\n  \"ts\"=\"2026-03-07 23:09:31.163568\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-a6eb9d68aa9cc22f7e78141cf691002056d2ca6a7d34a8fba7d067b26fef9e86.yaml.gz\"\n  \"ts\"=\"2026-03-07 23:09:31.163634\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-07 23:09:31.164640\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-10\"\n  \"ts\"=\"2026-03-07 23:09:31.164656\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"arm64-vm-cluster-b4f8bt\"\n  \"ts\"=\"2026-03-07 23:09:31.407942\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-10\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-07 23:09:31.474217\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"arm64-vm-cluster-b4f8bt\" \"deployment\"=\"pooledNonMsiScopedAssignments-6htyoibevj7ye\"\n  \"ts\"=\"2026-03-07 23:09:31.546551\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-10\" \"deployment\"=\"msi-deployment-10\"\n  \"ts\"=\"2026-03-07 23:09:31.617334\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"arm64-vm-cluster-b4f8bt\" \"deployment\"=\"customer-infra-arm64-vm-hcp-cluster-qqhxlg\"\n  \"ts\"=\"2026-03-07 23:09:31.781469\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-07 23:09:31.781570\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-07 23:09:31.781667\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"arm64-vm-cluster-b4f8bt\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command",".go:284\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func15.2({0x7fb680243860, 0xc0007a8ba0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n                | \n                | By(\"waiting for first cluster to complete creation\")\n                \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n                | \tFrequency: framework.StandardPollInterval,\n                | })\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8229900?, 0xc0007a8ba0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 104\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/07/26 23:09:29.212\n  \"ts\"=\"2026-03-07 23:09:29.212333\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-07 23:09:29.212391\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-b8lhcv\" \"resourceGroup\"=\"customer-rg-rdfw7z\"\n  \"ts\"=\"2026-03-07 23:09:29.368867\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-b8lhcv\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-07 23:09:30.401204\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-w6dnnl\" \"resourceGroup\"=\"customer-rg-rdfw7z\"\n  \"ts\"=\"2026-03-07 23:09:30.553065\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-w6dnnl\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-07 23:09:31.704626\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-07 23:09:31.704730\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-07 23:09:31.704790\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-07 23:09:31.705908\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-98\"\n  \"ts\"=\"2026-03-07 23:09:31.705919\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-14\"\n  \"ts\"=\"2026-03-07 23:09:31.705964\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-147\"\n  \"ts\"=\"2026-03-07 23:09:31.705944\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-rdfw7z\"\n  \"ts\"=\"2026-03-07 23:09:31.851105\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-147\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-07 23:09:31.862895\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-14\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-07 23:09:31.897487\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-rdfw7z\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n  \"ts\"=\"2026-03-07 23:09:31.930676\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-98\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-07 23:09:31.979247\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-147\" \"deployment\"=\"msi-deployment-147\"\n  \"ts\"=\"2026-03-07 23:09:31.991987\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-14\" \"deployment\"=\"msi-deployment-14\"\n  \"ts\"=\"2026-03-07 23:09:32.065912\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-98\" \"deployment\"=\"msi-deployment-98\"\n  \"ts\"=\"2026-03-07 23:09:32.075509\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-rdfw7z\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-w6dnnl\"\n  \"ts\"=\"2026-03-07 23:09:32.444727\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-rdfw7z\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-03-07 23:09:32.605189\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-rdfw7z\" \"deployment\"=\"customer-infra-basic-hcp-cluster-b8lhcv\"\n  \"ts\"=\"2026-03-07 23:09:32.764560\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-07 23:09:32.764662\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-07 23:09:32.764749\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-rdfw7z\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n.go:284\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func15.2({0x7fb680243860, 0xc0007a8ba0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:126\n                | \n                | By(\"waiting for first cluster to complete creation\")\n                \u003e _, err = poller1.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n                | \tFrequency: framework.StandardPollInterval,\n                | })\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8229900?, 0xc0007a8ba0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 104\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:129 @ 03/07/26 23:09:29.212\n  \"ts\"=\"2026-03-07 23:09:29.212333\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-07 23:09:29.212391\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-b8lhcv\" \"resourceGroup\"=\"customer-rg-rdfw7z\"\n  \"ts\"=\"2026-03-07 23:09:29.368867\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-b8lhcv\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-07 23:09:30.401204\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-w6dnnl\" \"resourceGroup\"=\"customer-rg-rdfw7z\"\n  \"ts\"=\"2026-03-07 23:09:30.553065\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-w6dnnl\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-07 23:09:31.704626\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-07 23:09:31.704730\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-07 23:09:31.704790\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-07 23:09:31.705908\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-98\"\n  \"ts\"=\"2026-03-07 23:09:31.705919\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-14\"\n  \"ts\"=\"2026-03-07 23:09:31.705964\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-147\"\n  \"ts\"=\"2026-03-07 23:09:31.705944\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-rdfw7z\"\n  \"ts\"=\"2026-03-07 23:09:31.851105\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-147\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-07 23:09:31.862895\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-14\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-07 23:09:31.897487\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-rdfw7z\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n  \"ts\"=\"2026-03-07 23:09:31.930676\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-98\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-07 23:09:31.979247\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-147\" \"deployment\"=\"msi-deployment-147\"\n  \"ts\"=\"2026-03-07 23:09:31.991987\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-14\" \"deployment\"=\"msi-deployment-14\"\n  \"ts\"=\"2026-03-07 23:09:32.065912\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-int-98\" \"deployment\"=\"msi-deployment-98\"\n  \"ts\"=\"2026-03-07 23:09:32.075509\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-rdfw7z\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-w6dnnl\"\n  \"ts\"=\"2026-03-07 23:09:32.444727\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-rdfw7z\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-03-07 23:09:32.605189\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-rdfw7z\" \"deployment\"=\"customer-infra-basic-hcp-cluster-b8lhcv\"\n  \"ts\"=\"2026-03-07 23:09:32.764560\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-07 23:09:32.764662\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-07 23:09:32.764749\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-rdfw7z\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2030395131827326976","occurred_at":"2026-03-07T21:28:16Z","signature_id":"36f13fa70accc7de33e434e01de801c434bbc56eae084fe5e7933a32526b4413","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2030395131827326976","occurred_at":"2026-03-07T21:28:16Z","signature_id":"b856009323f484ce23d7d9608919112065de23e933158cd5b75467a071eacfda","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2030395131827326976","occurred_at":"2026-03-07T21:28:16Z","signature_id":"f0057733f37443d7f7fc99cdc0392fdad8755ec05550f97db1b34c5d995c84e6","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"2d1906ec85d934b5bfe9201fdd489d4450037751378f0d66a8abd6c1b1f5a4bd","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":7,"trend_counts":[2,0,0,0,0,0,1],"trend_range":"2026-03-01..2026-03-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-02-23"],"prior_runs_affected":7,"prior_last_seen_at":"2026-03-01T04:25:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:157]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2030395131827326976","occurred_at":"2026-03-07T21:28:16Z","signature_id":"b8285d1fe5e0e06b0118077c74385230269c41bde5a189b9513acacc0d1a338e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"d6d90bfef9e01d3e5974032d0586fabac088b74b34d0095d1e8598be9d65dfae","failure_pattern":"ERROR CODE: Conflict; detail message Cannot update resource while resource is \"accepted\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-03-01..2026-03-07 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:90]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009c8220\u003e: \n    failed to create HCP cluster basic-hcp-cluster: failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-lpnmcl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-lpnmcl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot update resource while resource is \\\"accepted\\\"\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-lpnmcl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster basic-hcp-cluster: failed starting cluster creation \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-lpnmcl\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-lpnmcl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Cannot update resource while resource is \\\\\\\"accepted\\\\\\\"\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-lpnmcl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0009c8200\u003e{\n            msg: \"failed starting cluster creation \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-lpnmcl\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-lpnmcl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Cannot update resource while resource is \\\\\\\"accepted\\\\\\\"\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-lpnmcl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00125e7e0\u003e{\n                ErrorCode: \"Conflict\",\n                StatusCode: 409,\n                RawResponse: {\n                    Status: \"409 Conflict\",\n                    StatusCode: 409,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Date\": [\n                            \"Sat, 07 Mar 2026 00:17:12 GMT\",\n                        ],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"UKSOUTH:20260307T001712Z:97607234-1d4e-40a3-b8e5-4aac73aa1e83\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 8E3A3DDB29E7437B8A99525E6E2E994C Ref B: CH1AA2020603053 Ref C: 2026-03-07T00:17:12Z\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"311\"],\n                        \"X-Ms-Request-Id\": [\n                            \"e1ecbf3e-3dc4-4bd6-b628-f10b7dfdb837\",\n                        ],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/a8f427fa-8c59-4633-83d9-4928d11132df\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"97607234-1d4e-40a3-b8e5-...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:90]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0009c8220\u003e: \n    failed to create HCP cluster basic-hcp-cluster: failed starting cluster creation \"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-lpnmcl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-lpnmcl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot update resource while resource is \\\"accepted\\\"\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-lpnmcl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to create HCP cluster basic-hcp-cluster: failed starting cluster creation \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-lpnmcl\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-lpnmcl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Cannot update resource while resource is \\\\\\\"accepted\\\\\\\"\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-lpnmcl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*fmt.wrapError | 0xc0009c8200\u003e{\n            msg: \"failed starting cluster creation \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-lpnmcl\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-lpnmcl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Cannot update resource while resource is \\\\\\\"accepted\\\\\\\"\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/basic-cluster-lpnmcl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/basic-hcp-cluster\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n            err: \u003c*exported.ResponseError | 0xc00125e7e0\u003e{\n                ErrorCode: \"Conflict\",\n                StatusCode: 409,\n                RawResponse: {\n                    Status: \"409 Conflict\",\n                    StatusCode: 409,\n                    Proto: \"HTTP/2.0\",\n                    ProtoMajor: 2,\n                    ProtoMinor: 0,\n                    Header: {\n                        \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                        \"Date\": [\n                            \"Sat, 07 Mar 2026 00:17:12 GMT\",\n                        ],\n                        \"Content-Type\": [\"application/json\"],\n                        \"X-Ms-Routing-Request-Id\": [\n                            \"UKSOUTH:20260307T001712Z:97607234-1d4e-40a3-b8e5-4aac73aa1e83\",\n                        ],\n                        \"X-Msedge-Ref\": [\n                            \"Ref A: 8E3A3DDB29E7437B8A99525E6E2E994C Ref B: CH1AA2020603053 Ref C: 2026-03-07T00:17:12Z\",\n                        ],\n                        \"Cache-Control\": [\"no-cache\"],\n                        \"Content-Length\": [\"311\"],\n                        \"X-Ms-Request-Id\": [\n                            \"e1ecbf3e-3dc4-4bd6-b628-f10b7dfdb837\",\n                        ],\n                        \"X-Ms-Operation-Identifier\": [\n                            \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=c209f8df-52ae-48fb-98ea-380f58b04652/uksouth/a8f427fa-8c59-4633-83d9-4928d11132df\",\n                        ],\n                        \"Strict-Transport-Security\": [\n                            \"max-age=31536000; includeSubDomains\",\n                        ],\n                        \"Pragma\": [\"no-cache\"],\n                        \"Expires\": [\"-1\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                        \"X-Ms-Correlation-Request-Id\": [\n                            \"97607234-1d4e-40a3-b8e5-...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2030074104035414016","occurred_at":"2026-03-07T00:12:37Z","signature_id":"74d3825dcb337e9707cc0d9a309088faf7b1ed93fe6afd7437544100682afb1c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"156b691244df26e8d7f43bbd2ff00370f632b560f797dd0a0b972d14b4223aff","failure_pattern":"failed waiting for deployment \"aro-hcp-demo\" in resourcegroup=\"\u003cresource-group\u003e\" to finish: context deadline exceeded","search_query":"failed waiting for deployment \"aro-hcp-demo\" in resourcegroup=\"gpu-nodepools-NC4asT4v3-z4g56q\" to finish: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes vm type NC4asT4v3 in a single cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-01..2026-03-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-02-23"],"prior_runs_affected":9,"prior_last_seen_at":"2026-03-01T04:25:41Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel","test_name":"HCP Nodepools GPU instances creates and deletes vm type NC4asT4v3 in a single cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:96]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0004ac420\u003e: \n    failed waiting for deployment \"aro-hcp-demo\" in resourcegroup=\"gpu-nodepools-NC4asT4v3-z4g56q\" to finish: context deadline exceeded\n    {\n        msg: \"failed waiting for deployment \\\"aro-hcp-demo\\\" in resourcegroup=\\\"gpu-nodepools-NC4asT4v3-z4g56q\\\" to finish: context deadline exceeded\",\n        err: \u003ccontext.deadlineExceededError\u003e{},\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:96]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0004ac420\u003e: \n    failed waiting for deployment \"aro-hcp-demo\" in resourcegroup=\"gpu-nodepools-NC4asT4v3-z4g56q\" to finish: context deadline exceeded\n    {\n        msg: \"failed waiting for deployment \\\"aro-hcp-demo\\\" in resourcegroup=\\\"gpu-nodepools-NC4asT4v3-z4g56q\\\" to finish: context deadline exceeded\",\n        err: \u003ccontext.deadlineExceededError\u003e{},\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-integration-e2e-parallel/2030395131827326976","occurred_at":"2026-03-07T21:28:16Z","signature_id":"52d7bd0f58e5a22c07510e1a1fa864ecdc077bb6b1ad31d83a60d14a9bd0e3ac","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}