{"meta":{"start_date":"2026-05-04","end_date":"2026-05-10","start_at":"2026-05-04T00:00:00Z","end_at":"2026-05-11T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T11:43:35Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":39,"failed_runs":11,"raw_occurrences":99,"matched_occurrences":99,"runs_affected":11},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-lpqgjm, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-lpqgjm\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-lpqgjm, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":81,"runs_affected":7,"failed_runs":7,"run_impact_percent":17.94871794871795,"anchor_occurrences":81,"after_last_push_count":81,"trend_counts":[0,32,0,0,12,37,0,0],"trend_range":"2026-05-03..2026-05-10 UTC","prior_windows_present":4,"prior_window_starts":["2026-04-06","2026-04-13","2026-04-20","2026-04-27"],"prior_runs_affected":35,"prior_last_seen_at":"2026-05-01T03:28:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":6},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000b66120\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-bf6dd9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group complex-cilium-kv-bf6dd9, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:104]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000b66120\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-bf6dd9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group complex-cilium-kv-bf6dd9, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016085d0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-nrstp7, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-nrstp7 in resource group rg-zstream-upgrade-4-21-dbw6vp, error: failed waiting for cluster=\"cluster-zstream-4-21-nrstp7\" in resourcegroup=\"rg-zstream-upgrade-4-21-dbw6vp\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-nrstp7 in resource group rg-zstream-upgrade-4-21-dbw6vp, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016085d0\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-nrstp7, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-nrstp7 in resource group rg-zstream-upgrade-4-21-dbw6vp, error: failed waiting for cluster=\"cluster-zstream-4-21-nrstp7\" in resourcegroup=\"rg-zstream-upgrade-4-21-dbw6vp\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-nrstp7 in resource group rg-zstream-upgrade-4-21-dbw6vp, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:126]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015ebe90\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-l8tbkk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-l8tbkk in resource group rg-cp-ystream-upgrade-4-21-9xcdts, error: failed waiting for cluster=\"cp-ystream-upgrade-4-21-l8tbkk\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-9xcdts\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-l8tbkk in resource group rg-cp-ystream-upgrade-4-21-9xcdts, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:126]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015ebe90\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-l8tbkk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-l8tbkk in resource group rg-cp-ystream-upgrade-4-21-9xcdts, error: failed waiting for cluster=\"cp-ystream-upgrade-4-21-l8tbkk\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-9xcdts\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-l8tbkk in resource group rg-cp-ystream-upgrade-4-21-9xcdts, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052809265785081856","occurred_at":"2026-05-08T17:54:02Z","signature_id":"630aa3c31463db2f0f2ace7285a7db34e22407723279c17a940b9e46ee56ac5a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052680800888950784","occurred_at":"2026-05-08T09:23:34Z","signature_id":"10258748e3a49564c758ee3d15623aae9c0d42231f0e44b0f6aa3604873ebf45","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052680800888950784","occurred_at":"2026-05-08T09:23:34Z","signature_id":"10bd7ba40f86874640c4582ede0e92ab2137c838789de6abd0e28ba9b9daf495","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052680800888950784","occurred_at":"2026-05-08T09:23:34Z","signature_id":"2e6ef4ee0237634874f798a7baac6c2ce2a28b48f6592d5c7cfd54579b871b38","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052680800888950784","occurred_at":"2026-05-08T09:23:34Z","signature_id":"42d0c6916e2d13266dfe209fc1ab901bfee7c331418fbcb18d423772e68c6e71","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052680800888950784","occurred_at":"2026-05-08T09:23:34Z","signature_id":"675a2c0e8f86c301d222c49991c97ef541d3e650c243de4346f06b5c8d9ed341","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052680800888950784","occurred_at":"2026-05-08T09:23:34Z","signature_id":"6b4fb8ca3b86ab5b4adaa06e87cdf92cfbaa8541771eef30559285c8b8567713","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052680800888950784","occurred_at":"2026-05-08T09:23:34Z","signature_id":"825a1844ab11992d11a9a6aff17bf3c7a8eb23ee162a6e8976a66369ef248064","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052680800888950784","occurred_at":"2026-05-08T09:23:34Z","signature_id":"86be4fa1b724f2f04d49a14bc6bb5c6ce0cc47462b22a906b19746913eaa941e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052680800888950784","occurred_at":"2026-05-08T09:23:34Z","signature_id":"91e08797c7b95af5939914c7dc23fd127a41b7f48154fc0928d8bf2cb6006d7a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052680800888950784","occurred_at":"2026-05-08T09:23:34Z","signature_id":"971ee98f059ef5b2dfe2b72d7732dcbef9a1849fd9722e93a6922b70d2c6c4d9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052680800888950784","occurred_at":"2026-05-08T09:23:34Z","signature_id":"99b8e540c9b200cbfd79a911114e422dea1bc0e9d1e83bd2515acdf295e714ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052680800888950784","occurred_at":"2026-05-08T09:23:34Z","signature_id":"bb6de434c13410a0e8e7ba7c9b5d9268300c73adb7c103a348e91eb73478f76f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052680800888950784","occurred_at":"2026-05-08T09:23:34Z","signature_id":"dd07fd16a7d19b4f434a4dab92128e18cb18775029252b61582c85242e122307","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052680800888950784","occurred_at":"2026-05-08T09:23:34Z","signature_id":"ed02752d4bff2b178d6c33755eaf12cf3de288aae48bbb18fe96e0275416b6a9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052627527997853696","occurred_at":"2026-05-08T05:51:53Z","signature_id":"0b7c6c380472cceaf71c8e68ceff25572b4020adf4a945cefc23c05322fa7185","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052627527997853696","occurred_at":"2026-05-08T05:51:53Z","signature_id":"24a61af71094f216d459d8c1da4e793b0b0fea0674e2aa35cb21dbc07d0ace35","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052627527997853696","occurred_at":"2026-05-08T05:51:53Z","signature_id":"2eded0edc6f920d82405fb1da27b7d64871e8afc79e8c602d481b23cf743675e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052627527997853696","occurred_at":"2026-05-08T05:51:53Z","signature_id":"633c6b4046fad7ad87cdd48e8959037239aebf8a4fc7ecb3d8bb365f1155a2c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052627527997853696","occurred_at":"2026-05-08T05:51:53Z","signature_id":"7144daab41e901301ea1ee21d7b149e97520d23470d4a948369181a0425b8991","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052627527997853696","occurred_at":"2026-05-08T05:51:53Z","signature_id":"b055e3539cf07e98b36504d1b626ad52bc637bd906e959693e10f4c45a57ddbc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052627527997853696","occurred_at":"2026-05-08T05:51:53Z","signature_id":"c94f04187fb9deb9fced30d24e6c28cba651944e736b9b5544a54401bc84d342","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052627527997853696","occurred_at":"2026-05-08T05:51:53Z","signature_id":"f7012427ba91c0f1d2a56a61e891a4a3bde7259680fb3fc3cf4f5ee7c7020355","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052576880476819456","occurred_at":"2026-05-08T02:30:37Z","signature_id":"0986511271b02ab9a8cbfd127d866d527267b78a1d73a4ede74cf777e74113ea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052576880476819456","occurred_at":"2026-05-08T02:30:37Z","signature_id":"17911e8a86fb6c51841f2ef68b4d9c86606606d00af54b5f65bc337b60585b97","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052576880476819456","occurred_at":"2026-05-08T02:30:37Z","signature_id":"2939a1cee2a914ee3c5a0a57d5183eee17690694178fe51eeaade91d16d85758","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052576880476819456","occurred_at":"2026-05-08T02:30:37Z","signature_id":"365b6d186b4a3fd1c14944cb1c5f860fd30acfc3930f3c63a1af05eda5c14f5a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052576880476819456","occurred_at":"2026-05-08T02:30:37Z","signature_id":"60f634fc3d07f43e55c0248a37670f486ceb8f762810fe4af450f4148060ac13","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052576880476819456","occurred_at":"2026-05-08T02:30:37Z","signature_id":"6755b2d69324e05350322381c69e73d84ba0999bdd69f6d4e4d733b3efd84ed7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052576880476819456","occurred_at":"2026-05-08T02:30:37Z","signature_id":"6821669c227bcdba84ce75c8fee5ea002a353493c65075a667842aea7aa09760","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052576880476819456","occurred_at":"2026-05-08T02:30:37Z","signature_id":"9d400636762149f1003486beadac59e27f3c7eb479528c718d1b495524ea0780","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052576880476819456","occurred_at":"2026-05-08T02:30:37Z","signature_id":"ac04d596e951bba2aa2d22afde7aab16a6a68f16dfb8121a9b967f541521a3b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052576880476819456","occurred_at":"2026-05-08T02:30:37Z","signature_id":"cf7d1385967c5fdf2625968f58af89e05744066c1095697a2bc04fd8ef55bde9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052576880476819456","occurred_at":"2026-05-08T02:30:37Z","signature_id":"d077a9e084922963654f2460f9d38045a186ff99bddd89e7b2e21f6437d97307","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052576880476819456","occurred_at":"2026-05-08T02:30:37Z","signature_id":"e63527f8a17ca4f24d870de1d6ae63453253048b967a9ad94044b501ae3eb782","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052576880476819456","occurred_at":"2026-05-08T02:30:37Z","signature_id":"e9cc227348272f613b196033b9be928e8c06bcb0e17bcf38b0078def89c63367","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2052569181903130624","occurred_at":"2026-05-08T02:00:02Z","signature_id":"b0f7bea0f9caea447eb9f8e0bd1ab26837db331a9fb46d34a2d911ec6f714914","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052503972110929920","occurred_at":"2026-05-07T21:40:55Z","signature_id":"1e97671a3e297f977c77f770f20b1015766a19b1bf1f71e856a7df652d1da37d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052503972110929920","occurred_at":"2026-05-07T21:40:55Z","signature_id":"200dec8c1a3afb79bc2be2558958d3d6e8b2d6343904e25146377277b242db9b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052503972110929920","occurred_at":"2026-05-07T21:40:55Z","signature_id":"41100447f1671f9bf75406ef36bb592228fdc1573b7f363c86f54d5d46c7dbd7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052503972110929920","occurred_at":"2026-05-07T21:40:55Z","signature_id":"459dea2e4adbcc48cca950aa9765834b42cde7521b482e58b429d832a1a3b9db","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052503972110929920","occurred_at":"2026-05-07T21:40:55Z","signature_id":"4713d3b2d514218a481738503eac0cacf4807b90e9bab88276a77e49ec46d8c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052503972110929920","occurred_at":"2026-05-07T21:40:55Z","signature_id":"47c8bd12a734dc183087ef7b7ff865875dfef311924f87c63b24f9e4abb91bf4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052503972110929920","occurred_at":"2026-05-07T21:40:55Z","signature_id":"b0b823e93665d2615cda572ec433cad9dee835053492ab21dc8e6a1c6b3644d9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052503972110929920","occurred_at":"2026-05-07T21:40:55Z","signature_id":"b4276debd3ff54240c0e4d9146d8d4331a3f67b57a4f61d6c49756c0b331bd6d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052503972110929920","occurred_at":"2026-05-07T21:40:55Z","signature_id":"c8d4f09c46843fe56210faf95e234365bd8c21dce1768299080c95fafe1a9250","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052503972110929920","occurred_at":"2026-05-07T21:40:55Z","signature_id":"d8fc7946778b4e494277e6c95ee37fe11e7573f474224a6235f4399d287a7258","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052503972110929920","occurred_at":"2026-05-07T21:40:55Z","signature_id":"de39a9d831aa799f42fc8300ca800bb03f8270304f59fb48850742def1b7ed8f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052503972110929920","occurred_at":"2026-05-07T21:40:55Z","signature_id":"fa55ca847941d805298ddf554f992f9491e09af561c1f621303b71024e1439a0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"2c39e2a0b9ca95b64e120a9777b7e727c4d444a3d39910d1ffc3e50c91c0ccc5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"2effd0dd9245d69950534e14315274dd0c4e47651d842f2fa567768fec5f81f6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"307c8dfa5c44b4c046e10ac7c10b099ce6d1cf84421b9b40a86ad7b3fb3ff033","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"3853932099ee297101bde4e6420c4fe4924b1aae1eaf27e8250f931d732aa2cc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"496e16d4b55ae4dd003a610c530ce46884067632a8d29704ddf7b0aef2664173","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"4ac193ee73c2b3e80c5d4819d6bfbfb8f3f649bd74133e8c1d7bd848a9d0dd0b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"58606da7ed773998e0f498622bcef17a0c3318787fb7b20b4dee4072516498fd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"5bc31052c1ebc8352b24d838dde3d0e459dc1cf8c1b5b468545b1ae52eca292c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"622149a6b72c2d6c3372521072f680013b7e9403c49f4078aa1f931b05001dbd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"642a9667487e54a998ffd0dafb8563eb138c81a0f4821f1ec02ba9e25cea5b75","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"6860aa46a0234329aa87553d8ee11a2625736d86a411f4be0b6ddb850d53b0e6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"7821b612a98e6dd446bbcc82630581057af6629b02807db008016a04a9a1d8e2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"836d3fc203b4d1549678516b437cd55c4463bd8b717a31d39453f7f4eeacafbd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"83939de355f8c4ad153560fc1927b7b00499d77daad5a5cabb7b742a282ff044","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"8458fb515dbf774427d0c68e68cba5cd20e77d61d6eff2e4fcd7875acc08dae9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"84f01ddae3385f044283562b6616796101e4519c2d10f687ec2817e026aed194","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"8a071f9102350fad6cb85c2fd56bd662e081449f5b34dbddcc067ba4437e17f2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"8eeb14b37d294ffe98c3aded0dacf1c2c543b333131f1c1de93baf6210a37b22","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"95ada3cd75c7f5efcb663b991723be4065ddb47751d6ca39e15f53f6951bcd99","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"998beeb09893fa243f26a38f40c4203361fbb4fd773741d35e210c88dce96aa4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"9a1fa41859bf92af77752195135c1dcd46702a433f6aaa73bf1b9abf9e65ba60","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"9ce383a8067834e96cf8a1836003a7ad5d20233d58ff58ad1e70b3a2aa48c99b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"ab604d83f2b8381b3a9bbed7beac9665621986b055361b56f49e8c50c004eb41","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"ac5915b0798a74efe429fc48c011fd5774fcf9c058489c2f4d3eb77225f2263a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"c42a0a9ced503f7e4ba9bd96c4899cef4008f42a686f7160d614737ed4f0ce43","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"c679c3e32a028d8e41ec9b74d070f551df7d1efd69cc87444bca14e9944e3125","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"cb743d59cd6845edd94ed4667c4c237a70bb1bc244c4537102487489d8240dae","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"ce61c1e8e008d08620277c8743894e19882835145297848b8b405c8b5e40cbcb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"d3e072f85248bb98f501776694fd1eed6f33832139bd0e5918d530a2ef96598d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"d471e4fe80c5552b08d146743f1f34c09ef5f3f7b8b76a9f0b6203219b96f51f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"e73c7e8496c79a99b4ea66a60558c08e331b989017c1d129d0d2651cd387c7c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"f7cdfdcf5f3ba93daa91093903f6948584ef5ab9637888c20e8b9b5ea4432889","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"9626adf85231cae7571909c00084bdfb7c9de3df3208a3d5200ee95baf8fc7e8","failure_pattern":"context deadline exceeded","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":10.256410256410257,"anchor_occurrences":4,"after_last_push_count":11,"trend_counts":[0,1,0,0,1,2,0,0],"trend_range":"2026-05-03..2026-05-10 UTC","prior_windows_present":2,"prior_window_starts":["2026-04-20","2026-04-27"],"prior_runs_affected":11,"prior_last_seen_at":"2026-04-30T02:00:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:132]: Unexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:132]: Unexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:139]: Unexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:139]: Unexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: Unexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: Unexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052680800888950784","occurred_at":"2026-05-08T09:23:34Z","signature_id":"b15c55d32190fb3d32f16f58a52f080055a52cddf394a6cd7e062c8467f788e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052576880476819456","occurred_at":"2026-05-08T02:30:37Z","signature_id":"b15c55d32190fb3d32f16f58a52f080055a52cddf394a6cd7e062c8467f788e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052503972110929920","occurred_at":"2026-05-07T21:40:55Z","signature_id":"b15c55d32190fb3d32f16f58a52f080055a52cddf394a6cd7e062c8467f788e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"b15c55d32190fb3d32f16f58a52f080055a52cddf394a6cd7e062c8467f788e4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":4,"after_last_push_count":19,"trend_counts":[0,4,0,0,0,0,0,0],"trend_range":"2026-05-03..2026-05-10 UTC","prior_windows_present":3,"prior_window_starts":["2026-04-06","2026-04-20","2026-04-27"],"prior_runs_affected":22,"prior_last_seen_at":"2026-04-30T02:00:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:199\n          \u003e github.com/onsi/ginkgo/v2/internal.(*group).run(0xc000edeb88, {0xc000e9e120?, 0x0?, 0x0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:349\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).runSpecs(0xc000609508, {0x0, 0x0}, {0xc3060e0, 0x0, 0x0}, {0xc00007a094, 0x2a}, 0x0, {0xc000e9e100, ...})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:496\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).RunSpec(_, {_, _}, {_, _, _}, {_, _}, {0xc00007a094, 0x2a}, ...)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite_patch.go:68\n            github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite.func1.1({0x2d4da85?, 0xc000e80000?})\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/ginkgo/util.go:85\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.runSpec({0x8e0d3a8, 0xc000c51b80}, 0xc000a81d40, 0x1)\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:596\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:237\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:230\n\n          goroutine 53 [select]\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2(0xc000e297a0?)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:131\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts in goroutine 98\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:128\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:115 @ 05/04/26 08:51:52.884\n  \"ts\"=\"2026-05-04 08:51:52.884562\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-05-04 08:51:52.884648\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-candidate-4-20-mnd8tb-7qrshq\" \"resourceGroup\"=\"rg-candidate-4-20-6dvgzr\"\n  \"ts\"=\"2026-05-04 08:51:52.983579\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-candidate-4-20-mnd8tb-7qrshq\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-05-04 08:51:53.753121\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-d4ee2620a775558818d65bdad60edec7ff8775b05730402bc0de88c0d1c3d86f.yaml.gz\"\n  \"ts\"=\"2026-05-04 08:51:53.753232\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-d4ee2620a775558818d65bdad60edec7ff8775b05730402bc0de88c0d1c3d86f.yaml.gz\"\n  \"ts\"=\"2026-05-04 08:51:53.753300\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-05-04 08:51:53.754635\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-116\"\n  \"ts\"=\"2026-05-04 08:51:53.754644\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-candidate-4-20-6dvgzr\"\n  \"ts\"=\"2026-05-04 08:51:54.081137\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-116\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-05-04 08:51:54.363563\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-116\" \"deployment\"=\"msi-deployment-116\"\n  \"ts\"=\"2026-05-04 08:51:55.107531\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-candidate-4-20-6dvgzr\" \"deployment\"=\"pooledNonMsiScopedAssignments-zy6fi42ffv3g4\"\n  \"ts\"=\"2026-05-04 08:51:55.268696\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-candidate-4-20-6dvgzr\" \"deployment\"=\"customer-infra-cluster-candidate-4-20-mnd8tb-7qrshq\"\n  \"ts\"=\"2026-05-04 08:51:55.328144\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-05-04 08:51:55.542585\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-05-04 08:51:55.542824\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-candidate-4-20-6dvgzr\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:199\n          \u003e github.com/onsi/ginkgo/v2/internal.(*group).run(0xc000edeb88, {0xc000e9e120?, 0x0?, 0x0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/group.go:349\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).runSpecs(0xc000609508, {0x0, 0x0}, {0xc3060e0, 0x0, 0x0}, {0xc00007a094, 0x2a}, 0x0, {0xc000e9e100, ...})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:496\n          \u003e github.com/onsi/ginkgo/v2/internal.(*Suite).RunSpec(_, {_, _}, {_, _, _}, {_, _}, {0xc00007a094, 0x2a}, ...)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite_patch.go:68\n            github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite.func1.1({0x2d4da85?, 0xc000e80000?})\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/ginkgo/util.go:85\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.runSpec({0x8e0d3a8, 0xc000c51b80}, 0xc000a81d40, 0x1)\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:596\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run.func2()\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:237\n            github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n              /go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260408205138-ec501c2bf4a5/pkg/extension/extensiontests/spec.go:230\n\n          goroutine 53 [select]\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2(0xc000e297a0?)\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:131\n          \u003e github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts in goroutine 98\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/interrupt_handler/interrupt_handler.go:128\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:115 @ 05/04/26 08:51:52.884\n  \"ts\"=\"2026-05-04 08:51:52.884562\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-05-04 08:51:52.884648\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-candidate-4-20-mnd8tb-7qrshq\" \"resourceGroup\"=\"rg-candidate-4-20-6dvgzr\"\n  \"ts\"=\"2026-05-04 08:51:52.983579\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-candidate-4-20-mnd8tb-7qrshq\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-05-04 08:51:53.753121\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-d4ee2620a775558818d65bdad60edec7ff8775b05730402bc0de88c0d1c3d86f.yaml.gz\"\n  \"ts\"=\"2026-05-04 08:51:53.753232\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-d4ee2620a775558818d65bdad60edec7ff8775b05730402bc0de88c0d1c3d86f.yaml.gz\"\n  \"ts\"=\"2026-05-04 08:51:53.753300\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-05-04 08:51:53.754635\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-116\"\n  \"ts\"=\"2026-05-04 08:51:53.754644\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-candidate-4-20-6dvgzr\"\n  \"ts\"=\"2026-05-04 08:51:54.081137\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-116\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-05-04 08:51:54.363563\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-116\" \"deployment\"=\"msi-deployment-116\"\n  \"ts\"=\"2026-05-04 08:51:55.107531\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-candidate-4-20-6dvgzr\" \"deployment\"=\"pooledNonMsiScopedAssignments-zy6fi42ffv3g4\"\n  \"ts\"=\"2026-05-04 08:51:55.268696\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-candidate-4-20-6dvgzr\" \"deployment\"=\"customer-infra-cluster-candidate-4-20-mnd8tb-7qrshq\"\n  \"ts\"=\"2026-05-04 08:51:55.328144\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-05-04 08:51:55.542585\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-05-04 08:51:55.542824\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-candidate-4-20-6dvgzr\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","Customer should be able to create an HCP cluster and custom node pool osDisk size (Spec Runtime: 2h29m59.872s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:35\n        In [It] (Node Runtime: 2h29m59.872s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:35\n          At [By Step] creating the HCP cluster (Step Runtime: 15m50.744s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:73\n\n          Spec Goroutine\n          goroutine 116 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8e0d3a8, 0xc000a00eb0}, 0x82e1752?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8e4a1e0, {0x8e0d3a8, 0xc000a00eb0}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8e0d418?, 0xc000f32150?}, {{0x8e25e48?, 0xc00063ee80?}, 0xc00180e520?}, 0xc00008b760, {0xc000aa5830, 0x13}, {0x8300ae2, 0x12}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:852\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0xc0014ca377, 0x7}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:346\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func9.2({0x7fb0a8517460, 0xc000f45f20})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:74\n                | \n                | By(\"creating the HCP cluster\")\n                \u003e err = tc.CreateHCPClusterFromParam(ctx,\n                | \tGinkgoLogr,\n                | \t*resourceGroup.Name,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8e25f20?, 0xc000f45f20?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 49\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80 @ 05/04/26 08:51:52.884\n  \"ts\"=\"2026-05-04 08:51:52.884414\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-05-04 08:51:52.884487\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-hcp-cluster-np-128-pkvjb9\" \"resourceGroup\"=\"clusternp128-sl625r\"\n  \"ts\"=\"2026-05-04 08:51:52.940234\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-hcp-cluster-np-128-pkvjb9\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-05-04 08:51:53.919548\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-a85d76e69c62fcc4973899bcc5957cf5e02900c7383d8dbab637c1fde0fca561.yaml.gz\"\n  \"ts\"=\"2026-05-04 08:51:53.919661\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-a85d76e69c62fcc4973899bcc5957cf5e02900c7383d8dbab637c1fde0fca561.yaml.gz\"\n  \"ts\"=\"2026-05-04 08:51:53.919714\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-05-04 08:51:53.921093\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-134\"\n  \"ts\"=\"2026-05-04 08:51:53.921102\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"clusternp128-sl625r\"\n  \"ts\"=\"2026-05-04 08:51:54.229005\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-134\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-05-04 08:51:54.503759\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-134\" \"deployment\"=\"msi-deployment-134\"\n  \"ts\"=\"2026-05-04 08:51:54.692153\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"clusternp128-sl625r\" \"deployment\"=\"pooledNonMsiScopedAssignments-3ljyjjx2n7gxo\"\n  \"ts\"=\"2026-05-04 08:51:54.751953\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"clusternp128-sl625r\" \"deployment\"=\"customer-infra-hcp-cluster-np-128-pkvjb9\"\n  \"ts\"=\"2026-05-04 08:51:54.802292\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-05-04 08:51:55.368728\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-05-04 08:51:55.369538\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"clusternp128-sl625r\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nCustomer should be able to create an HCP cluster and custom node pool osDisk size (Spec Runtime: 2h29m59.872s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:35\n        In [It] (Node Runtime: 2h29m59.872s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:35\n          At [By Step] creating the HCP cluster (Step Runtime: 15m50.744s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:73\n\n          Spec Goroutine\n          goroutine 116 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8e0d3a8, 0xc000a00eb0}, 0x82e1752?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8e4a1e0, {0x8e0d3a8, 0xc000a00eb0}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.1/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8e0d418?, 0xc000f32150?}, {{0x8e25e48?, 0xc00063ee80?}, 0xc00180e520?}, 0xc00008b760, {0xc000aa5830, 0x13}, {0x8300ae2, 0x12}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:852\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0xc0014ca377, 0x7}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:346\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func9.2({0x7fb0a8517460, 0xc000f45f20})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:74\n                | \n                | By(\"creating the HCP cluster\")\n                \u003e err = tc.CreateHCPClusterFromParam(ctx,\n                | \tGinkgoLogr,\n                | \t*resourceGroup.Name,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8e25f20?, 0xc000f45f20?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 49\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80 @ 05/04/26 08:51:52.884\n  \"ts\"=\"2026-05-04 08:51:52.884414\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-05-04 08:51:52.884487\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-hcp-cluster-np-128-pkvjb9\" \"resourceGroup\"=\"clusternp128-sl625r\"\n  \"ts\"=\"2026-05-04 08:51:52.940234\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-hcp-cluster-np-128-pkvjb9\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-05-04 08:51:53.919548\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-a85d76e69c62fcc4973899bcc5957cf5e02900c7383d8dbab637c1fde0fca561.yaml.gz\"\n  \"ts\"=\"2026-05-04 08:51:53.919661\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-a85d76e69c62fcc4973899bcc5957cf5e02900c7383d8dbab637c1fde0fca561.yaml.gz\"\n  \"ts\"=\"2026-05-04 08:51:53.919714\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-05-04 08:51:53.921093\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-134\"\n  \"ts\"=\"2026-05-04 08:51:53.921102\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"clusternp128-sl625r\"\n  \"ts\"=\"2026-05-04 08:51:54.229005\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-134\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-05-04 08:51:54.503759\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-134\" \"deployment\"=\"msi-deployment-134\"\n  \"ts\"=\"2026-05-04 08:51:54.692153\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"clusternp128-sl625r\" \"deployment\"=\"pooledNonMsiScopedAssignments-3ljyjjx2n7gxo\"\n  \"ts\"=\"2026-05-04 08:51:54.751953\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"clusternp128-sl625r\" \"deployment\"=\"customer-infra-hcp-cluster-np-128-pkvjb9\"\n  \"ts\"=\"2026-05-04 08:51:54.802292\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-05-04 08:51:55.368728\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-05-04 08:51:55.369538\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"clusternp128-sl625r\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","x8e25f20?, 0xc000957470?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:293 +0x105 fp=0xc000a99f88 sp=0xc000a99f18 pc=0x3243aa5\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000a99fe0 sp=0xc000a99f88 pc=0x323541b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000a99fe8 sp=0xc000a99fe0 pc=0x2dc6cc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 103\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\ngoroutine 165 gp=0xc000582fc0 m=nil [IO wait]:\nruntime.gopark(0xc000127900?, 0xfb7c2708?, 0x0?, 0xb9?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000e4e720 sp=0xc000e4e700 pc=0x2dbe50e\nruntime.netpollblock(0x2e28d18?, 0x2d4a066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc000e4e758 sp=0xc000e4e720 pc=0x2d7dfb7\ninternal/poll.runtime_pollWait(0x7f6a088ac600, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc000e4e778 sp=0xc000e4e758 pc=0x2dbcda5\ninternal/poll.(*pollDesc).wait(0xc000801500?, 0xc0011aa000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc000e4e7a0 sp=0xc000e4e778 pc=0x2e4bb47\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc000801500, {0xc0011aa000, 0x3000, 0x3000})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc000e4e838 sp=0xc000e4e7a0 pc=0x2e4cf79\nnet.(*netFD).Read(0xc000801500, {0xc0011aa000?, 0xc0011aa302?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc000e4e880 sp=0xc000e4e838 pc=0x3010ea5\nnet.(*conn).Read(0xc000e02038, {0xc0011aa000?, 0x7f69fb7dfc20?, 0x7f6a50cd5108?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc000e4e8c8 sp=0xc000e4e880 pc=0x3022f45\ncrypto/tls.(*atLeastReader).Read(0xc001040108, {0xc0011aa000?, 0x2cf9?, 0x2dbb245?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc000e4e910 sp=0xc000e4e8c8 pc=0x308eadb\nbytes.(*Buffer).ReadFrom(0xc0004c89a8, {0x8d7a940, 0xc001040108})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc000e4e968 sp=0xc000e4e910 pc=0x2e8a3d8\ncrypto/tls.(*Conn).readFromUntil(0xc0004c8708, {0x8d7b3e0, 0xc000e02038}, 0xc000e4e9d0?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc000e4e9a0 sp=0xc000e4e968 pc=0x308ecbe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc0004c8708, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc000e4ec18 sp=0xc000e4e9a0 pc=0x308bd3b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc0004c8708, {0xc001047000, 0x1000, 0x348ce60?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc000e4ec80 sp=0xc000e4ec18 pc=0x3092645\nbufio.(*Reader).Read(0xc0010442a0, {0xc0006ea824, 0x9, 0x349aeee?})\n\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0xc000e4ecb8 sp=0xc000e4ec80 pc=0x2f24a97\nio.ReadAtLeast({0x8d79da0, 0xc0010442a0}, {0xc0006ea824, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0xc000e4ed00 sp=0xc000e4ecb8 pc=0x2e18e8e\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc0006ea824, 0x9, 0xc0000000fb?}, {0x8d79da0?, 0xc0010442a0?})\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:250 +0x65 fp=0xc000e4ed50 sp=0xc000e4ed00 pc=0x3483c25\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc0006ea7e0)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:513 +0x6b fp=0xc000e4edd8 sp=0xc000e4ed50 pc=0x34847eb\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc0006ea7e0)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:572 +0x18 fp=0xc000e4ee18 sp=0xc000e4edd8 pc=0x3484f58\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000e4efa8)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2251 +0xca fp=0xc000e4ef60 sp=0xc000e4ee18 pc=0x3497e2a\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc000583880)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2120 +0x52 fp=0xc000e4efc8 sp=0xc000e4ef60 pc=0x34971b2\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0x25 fp=0xc000e4efe0 sp=0xc000e4efc8 pc=0x3490945\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000e4efe8 sp=0xc000e4efe0 pc=0x2dc6cc1\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 164\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0xda5\n\nrax    0xfffffffffffffffc\nrbx    0x0\nrcx    0x2dc8ae3\nrdx    0x0\nrdi    0xc2dd4d0\nrsi    0x80\nrbp    0x7f6a0a0e1cb8\nrsp    0x7f6a0a0e1c70\nr8     0x0\nr9     0x0\nr10    0x7f6a0a0e1ca8\nr11    0x202\nr12    0x7f6a0a0e1ca8\nr13    0x1\nr14    0xc000002c40\nr15    0x0\nrip    0x2dc8ae3\nrflags 0x202\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\nx8e25f20?, 0xc000957470?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:293 +0x105 fp=0xc000a99f88 sp=0xc000a99f18 pc=0x3243aa5\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000a99fe0 sp=0xc000a99f88 pc=0x323541b\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000a99fe8 sp=0xc000a99fe0 pc=0x2dc6cc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 103\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\ngoroutine 165 gp=0xc000582fc0 m=nil [IO wait]:\nruntime.gopark(0xc000127900?, 0xfb7c2708?, 0x0?, 0xb9?, 0xb?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000e4e720 sp=0xc000e4e700 pc=0x2dbe50e\nruntime.netpollblock(0x2e28d18?, 0x2d4a066?, 0x0?)\n\t/usr/lib/golang/src/runtime/netpoll.go:575 +0xf7 fp=0xc000e4e758 sp=0xc000e4e720 pc=0x2d7dfb7\ninternal/poll.runtime_pollWait(0x7f6a088ac600, 0x72)\n\t/usr/lib/golang/src/runtime/netpoll.go:351 +0x85 fp=0xc000e4e778 sp=0xc000e4e758 pc=0x2dbcda5\ninternal/poll.(*pollDesc).wait(0xc000801500?, 0xc0011aa000?, 0x0)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc000e4e7a0 sp=0xc000e4e778 pc=0x2e4bb47\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/lib/golang/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc000801500, {0xc0011aa000, 0x3000, 0x3000})\n\t/usr/lib/golang/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc000e4e838 sp=0xc000e4e7a0 pc=0x2e4cf79\nnet.(*netFD).Read(0xc000801500, {0xc0011aa000?, 0xc0011aa302?, 0x5?})\n\t/usr/lib/golang/src/net/fd_posix.go:68 +0x25 fp=0xc000e4e880 sp=0xc000e4e838 pc=0x3010ea5\nnet.(*conn).Read(0xc000e02038, {0xc0011aa000?, 0x7f69fb7dfc20?, 0x7f6a50cd5108?})\n\t/usr/lib/golang/src/net/net.go:196 +0x45 fp=0xc000e4e8c8 sp=0xc000e4e880 pc=0x3022f45\ncrypto/tls.(*atLeastReader).Read(0xc001040108, {0xc0011aa000?, 0x2cf9?, 0x2dbb245?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:819 +0x3b fp=0xc000e4e910 sp=0xc000e4e8c8 pc=0x308eadb\nbytes.(*Buffer).ReadFrom(0xc0004c89a8, {0x8d7a940, 0xc001040108})\n\t/usr/lib/golang/src/bytes/buffer.go:217 +0x98 fp=0xc000e4e968 sp=0xc000e4e910 pc=0x2e8a3d8\ncrypto/tls.(*Conn).readFromUntil(0xc0004c8708, {0x8d7b3e0, 0xc000e02038}, 0xc000e4e9d0?)\n\t/usr/lib/golang/src/crypto/tls/conn.go:841 +0xde fp=0xc000e4e9a0 sp=0xc000e4e968 pc=0x308ecbe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc0004c8708, 0x0)\n\t/usr/lib/golang/src/crypto/tls/conn.go:630 +0x3db fp=0xc000e4ec18 sp=0xc000e4e9a0 pc=0x308bd3b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/lib/golang/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc0004c8708, {0xc001047000, 0x1000, 0x348ce60?})\n\t/usr/lib/golang/src/crypto/tls/conn.go:1397 +0x145 fp=0xc000e4ec80 sp=0xc000e4ec18 pc=0x3092645\nbufio.(*Reader).Read(0xc0010442a0, {0xc0006ea824, 0x9, 0x349aeee?})\n\t/usr/lib/golang/src/bufio/bufio.go:245 +0x197 fp=0xc000e4ecb8 sp=0xc000e4ec80 pc=0x2f24a97\nio.ReadAtLeast({0x8d79da0, 0xc0010442a0}, {0xc0006ea824, 0x9, 0x9}, 0x9)\n\t/usr/lib/golang/src/io/io.go:335 +0x8e fp=0xc000e4ed00 sp=0xc000e4ecb8 pc=0x2e18e8e\nio.ReadFull(...)\n\t/usr/lib/golang/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc0006ea824, 0x9, 0xc0000000fb?}, {0x8d79da0?, 0xc0010442a0?})\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:250 +0x65 fp=0xc000e4ed50 sp=0xc000e4ed00 pc=0x3483c25\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc0006ea7e0)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:513 +0x6b fp=0xc000e4edd8 sp=0xc000e4ed50 pc=0x34847eb\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc0006ea7e0)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/frame.go:572 +0x18 fp=0xc000e4ee18 sp=0xc000e4edd8 pc=0x3484f58\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000e4efa8)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2251 +0xca fp=0xc000e4ef60 sp=0xc000e4ee18 pc=0x3497e2a\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc000583880)\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:2120 +0x52 fp=0xc000e4efc8 sp=0xc000e4ef60 pc=0x34971b2\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0x25 fp=0xc000e4efe0 sp=0xc000e4efc8 pc=0x3490945\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000e4efe8 sp=0xc000e4efe0 pc=0x2dc6cc1\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 164\n\t/go/pkg/mod/golang.org/x/net@v0.53.0/http2/transport.go:873 +0xda5\n\nrax    0xfffffffffffffffc\nrbx    0x0\nrcx    0x2dc8ae3\nrdx    0x0\nrdi    0xc2dd4d0\nrsi    0x80\nrbp    0x7f6a0a0e1cb8\nrsp    0x7f6a0a0e1c70\nr8     0x0\nr9     0x0\nr10    0x7f6a0a0e1ca8\nr11    0x202\nr12    0x7f6a0a0e1ca8\nr13    0x1\nr14    0xc000002c40\nr15    0x0\nrip    0x2dc8ae3\nrflags 0x202\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"605a0197b4713db8b8f84ef541971c22751ce0fb3b16fb97c098151d2afb0c01","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"76329cf8e1fdcadecab6b2623ee9f3b2ca558dbf59aaa410a65ac76434c9d810","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"910891fab74051881ef979109795f23592af510dd8b2d0acce59ccdc7a257fa0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051181894506123264","occurred_at":"2026-05-04T06:07:27Z","signature_id":"947d536583f9ee39787f502a3d3296be500fb88f4c73ca5d0674ab556635a5b8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-6bh759, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-6bh759\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":7.6923076923076925,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,1,2,0],"trend_range":"2026-05-03..2026-05-10 UTC","prior_windows_present":4,"prior_window_starts":["2026-04-06","2026-04-13","2026-04-20","2026-04-27"],"prior_runs_affected":12,"prior_last_seen_at":"2026-04-30T03:16:57Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008f2540\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-6bh759, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-6bh759\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008f2540\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-6bh759, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-6bh759\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00120a870\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-wnqwj6, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-wnqwj6\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00120a870\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-wnqwj6, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-wnqwj6\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00170a570\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-8mnv7f, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-8mnv7f\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00170a570\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-8mnv7f, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-8mnv7f\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2053041571653226496","occurred_at":"2026-05-09T09:16:57Z","signature_id":"f6c68094dd85758aee0ec39e54a1e6f815d6f31823bddab256c0d9b9d2569570","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2052931676501184512","occurred_at":"2026-05-09T02:00:27Z","signature_id":"b35c2bdd9373d6f3607c9f6927cd978a5d4d88f4a57ee6c6d9b1be8d7cc872d7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052809265785081856","occurred_at":"2026-05-08T17:54:02Z","signature_id":"cf12f201eedcc16bc0bb21591ac2e9d17e933f099d2d566665049402f0a55d4c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":9,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-05-03..2026-05-10 UTC","prior_windows_present":3,"prior_window_starts":["2026-04-06","2026-04-20","2026-04-27"],"prior_runs_affected":10,"prior_last_seen_at":"2026-04-30T02:00:43Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:162]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052503972110929920","occurred_at":"2026-05-07T21:40:55Z","signature_id":"0c4ed9bba8fe112ea7971d2241bf4fe4d2d0df6edae20700d459cff3d63baf26","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"821a85980f58e3beddad169d86fa2f7b5125c71dfe036d6084a659edfb8d4b63","failure_pattern":"ERROR CODE: AuthenticationFailed; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: AuthenticationFailed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-05-03..2026-05-10 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:108]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000e04260\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-jc8js6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1555f87d-c77a-446e-9812-a74076b41278\n    --------------------------------------------------------------------------------\n    RESPONSE 401: 401 Unauthorized\n    ERROR CODE: AuthenticationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"AuthenticationFailed\",\n        \"message\": \"Authentication failed.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:108]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000e04260\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-jc8js6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1555f87d-c77a-446e-9812-a74076b41278\n    --------------------------------------------------------------------------------\n    RESPONSE 401: 401 Unauthorized\n    ERROR CODE: AuthenticationFailed\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"AuthenticationFailed\",\n        \"message\": \"Authentication failed.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052503972110929920","occurred_at":"2026-05-07T21:40:55Z","signature_id":"2e00571f16961e077f3d3694a6b40db810eb0db129137b333a52b8b23275a10e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-05-03..2026-05-10 UTC","prior_windows_present":3,"prior_window_starts":["2026-04-06","2026-04-20","2026-04-27"],"prior_runs_affected":20,"prior_last_seen_at":"2026-04-29T08:51:26Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052680800888950784","occurred_at":"2026-05-08T09:23:34Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":7,"trend_counts":[0,0,0,0,0,0,1,0],"trend_range":"2026-05-03..2026-05-10 UTC","prior_windows_present":1,"prior_window_starts":["2026-04-20"],"prior_runs_affected":7,"prior_last_seen_at":"2026-04-23T08:28:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc0009943d8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9wx5kr\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc0009943d8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9wx5kr\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2053041571653226496","occurred_at":"2026-05-09T09:16:57Z","signature_id":"3232292372eee42b391df8b0f55a0b6df00678af180346b2f11741f77140ba1e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"314b58d43728f942111a52209bcc085b0bf14adcab05fa6fc6959abcb89c83c4","failure_pattern":"dial tcp: lookup console-openshift-console","search_query":"dial tcp: lookup console-openshift-console","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-05-03..2026-05-10 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:153]: Timed out after 240.001s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003c*net.OpError | 0xc0008fa690\u003e: \n    dial tcp: lookup console-openshift-console.apps.aro.c1m2s6r4u2z7j9m.cpvn.centralindia.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:153]: Timed out after 240.001s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003c*net.OpError | 0xc0008fa690\u003e: \n    dial tcp: lookup console-openshift-console.apps.aro.c1m2s6r4u2z7j9m.cpvn.centralindia.aroapp-hcp.io on 172.30.0.10:53: no such host\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052369690725978112","occurred_at":"2026-05-07T12:47:20Z","signature_id":"ed04e2256a551b83782eeb9e0d5b43ec5a5d8ec7202e948007f18c4b0e3e56c1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f27b30bef0ebb941cd79b083d3eb5ff3005b4b85f92582e4349d8748b1eaeb94","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-hkqvxb\" to finish deleting: context canceled","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-05-03..2026-05-10 UTC","prior_windows_present":1,"prior_window_starts":["2026-04-20"],"prior_runs_affected":2,"prior_last_seen_at":"2026-04-24T18:17:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:249] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0011a2138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-hkqvxb\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:249] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0011a2138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-hkqvxb\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2052503972110929920","occurred_at":"2026-05-07T21:40:55Z","signature_id":"5a20f339b7d8b2aef04c5428a50fdf6772657e37db62a2b3458e4f0be17df4e7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"3abd48de63dd3956140a30196e0f3dc44998cdba01a694484355c666029b9eeb","failure_pattern":"subscription openshift-nfd/nfd not ready, state=\"\" (OLM still processing, no errors detected)","search_query":"subscription openshift-nfd/nfd not ready, state=\"\" (OLM still processing, no errors detected)","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.5641025641025643,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,1,0,0,0,0,0],"trend_range":"2026-05-03..2026-05-10 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:61]: Timed out after 600.005s.\nNFD operator should be installed successfully\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00137c5e0\u003e: \n    subscription openshift-nfd/nfd not ready, state=\"\" (OLM still processing, no errors detected)\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:61]: Timed out after 600.005s.\nNFD operator should be installed successfully\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc00137c5e0\u003e: \n    subscription openshift-nfd/nfd not ready, state=\"\" (OLM still processing, no errors detected)\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2051571821928517632","occurred_at":"2026-05-05T07:56:53Z","signature_id":"d2c9ad952198495db4eeedd9a6d906a865673fb528ca1ef6a6d54f670c0dae04","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}