{"meta":{"start_date":"2026-02-27","end_date":"2026-02-27","start_at":"2026-02-27T00:00:00Z","end_at":"2026-02-28T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T22:29:03Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":1,"failed_runs":1,"raw_occurrences":12,"matched_occurrences":12,"runs_affected":1},"rows":[{"environment":"prod","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-rkpqpt, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-rkpqpt\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,0,5],"trend_range":"2026-02-21..2026-02-27 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:124]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011d2e70\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-h5r2qd, error: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-h5r2qd\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-h5r2qd, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-h5r2qd\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00068ba00\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-h5r2qd\",\n            },\n            \u003c*fmt.wrapError | 0xc000b8ad80\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-h5r2qd\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:124]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011d2e70\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-h5r2qd, error: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-h5r2qd\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-h5r2qd, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-h5r2qd\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00068ba00\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-h5r2qd\",\n            },\n            \u003c*fmt.wrapError | 0xc000b8ad80\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-h5r2qd\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:94]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014baa20\u003e: \n    failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-xkk5mj, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-xkk5mj\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-xkk5mj, error: failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-xkk5mj\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0010e7c20\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-xkk5mj\",\n            },\n            \u003c*fmt.wrapError | 0xc0013b44a0\u003e{\n                msg: \"failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-xkk5mj\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:94]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014baa20\u003e: \n    failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-xkk5mj, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-xkk5mj\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-xkk5mj, error: failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-xkk5mj\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0010e7c20\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-xkk5mj\",\n            },\n            \u003c*fmt.wrapError | 0xc0013b44a0\u003e{\n                msg: \"failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-xkk5mj\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:242]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00176eae0\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-dcdzqf, error: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-dcdzqf\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-dcdzqf, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-dcdzqf\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001099d80\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-dcdzqf\",\n            },\n            \u003c*fmt.wrapError | 0xc000a7a9c0\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-dcdzqf\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:242]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00176eae0\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-dcdzqf, error: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-dcdzqf\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-dcdzqf, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-dcdzqf\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001099d80\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-dcdzqf\",\n            },\n            \u003c*fmt.wrapError | 0xc000a7a9c0\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-dcdzqf\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027202235628261376","occurred_at":"2026-02-27T02:00:51Z","signature_id":"1a9e4c88431060503f2ae3e726ffd75c99d22267aa6ad7ee7aef0b3b77cafd74","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027202235628261376","occurred_at":"2026-02-27T02:00:51Z","signature_id":"6af55799f6b3196aafa3f8b634ddafd8b0521cde1bc6dcc979bf4f328aa2ef59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027202235628261376","occurred_at":"2026-02-27T02:00:51Z","signature_id":"8012d4143da2e24909e7074f9e5021a126b83b97bf19484d0f01fcdabce73853","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027202235628261376","occurred_at":"2026-02-27T02:00:51Z","signature_id":"8332d59e9e3bfdd637e576495c32f2f0fc35dffabda71540834cb0cdcc87e0c0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027202235628261376","occurred_at":"2026-02-27T02:00:51Z","signature_id":"efd2dffe55331958aed415ee335a063c0372a7a4338470ac06517163de34cce9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-b22djs\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-b22djs, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-02-21..2026-02-27 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:156]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00118aa20\u003e: \n    failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-fqjfbz\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-fqjfbz, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-fqjfbz\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-fqjfbz, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0008bb430\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-fqjfbz\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:156]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00118aa20\u003e: \n    failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-fqjfbz\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-fqjfbz, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-fqjfbz\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-fqjfbz, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0008bb430\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-fqjfbz\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:100]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00119a8d0\u003e: \n    failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-b22djs\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-b22djs, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-b22djs\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-b22djs, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000646250\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-b22djs\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:100]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00119a8d0\u003e: \n    failed waiting for hcpCluster=\"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-b22djs\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-b22djs, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-b22djs\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-b22djs, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000646250\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster basic-hcp-cluster in resource group basic-cluster-b22djs\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e9b110\u003e: \n    failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-rxn4xb\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-rxn4xb, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-rxn4xb\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-rxn4xb, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000812100\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-rxn4xb\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e9b110\u003e: \n    failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-rxn4xb\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-rxn4xb, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-rxn4xb\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-rxn4xb, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000812100\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-rxn4xb\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027202235628261376","occurred_at":"2026-02-27T02:00:51Z","signature_id":"5b3391a38268c14609cca30cea107533ac00ae57a0ebacc3fec48c6dd66c2e12","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027202235628261376","occurred_at":"2026-02-27T02:00:51Z","signature_id":"7a5c855294d4b7559a091eec9d6730f869bac23a6af277628e120f883c2d7bfb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027202235628261376","occurred_at":"2026-02-27T02:00:51Z","signature_id":"a92636cec887bc776c99a3a0b7bac32c0f45af9f2ad616ce100f8a8dc03abd3d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-02-21..2026-02-27 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":[": 1h29m59.786s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:39\n        In [It] (Node Runtime: 1h29m59.786s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:39\n          At [By Step] creating the node pool (Step Runtime: 43m10.278s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:103\n\n          Spec Goroutine\n          goroutine 112 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x820b858, 0xc000dd0550}, 0x77f1e58?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x824f540, {0x820b858, 0xc000dd0550}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateNodePoolAndWait({0x820b8c8, 0xc000705490}, 0xc0000976f8, {0xc0015469c0, 0x1b}, {0x782be91, 0x18}, {0x77d2568, 0x4}, {0xc0017f1a80, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:718\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateNodePoolFromParam(0xc000deb8c0, {0x7f2a7c514b98, 0xc000a42540}, {0xc0015469c0, 0x1b}, {0x782be91, 0x18}, {{0x77d5c37, 0x6}, {0x782be91, ...}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:384\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func13.2({0x7f2a7c514b98, 0xc000a42540})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:108\n                | nodePoolParams.NodePoolName = customerNodePoolName\n                | \n                \u003e err = tc.CreateNodePoolFromParam(ctx,\n                | \t*resourceGroup.Name,\n                | \tcustomerClusterName,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82210c0?, 0xc000a42540?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 109\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:114 @ 02/27/26 03:34:05.194\n  \"ts\"=\"2026-02-27 03:34:05.194982\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-02-27 03:34:05.195051\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-tls-endpoint-hcp-cluster-t8phmj\" \"resourceGroup\"=\"tls-endpoint-cluster-8p26lt\"\n  \"ts\"=\"2026-02-27 03:34:05.722278\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-tls-endpoint-hcp-cluster-t8phmj\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-02-27 03:34:07.361055\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-81f205190df882aeed9c971ba5a51da44c00eb526b484755681d39236b7f3906.yaml.gz\"\n  \"ts\"=\"2026-02-27 03:34:07.361174\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-81f205190df882aeed9c971ba5a51da44c00eb526b484755681d39236b7f3906.yaml.gz\"\n  \"ts\"=\"2026-02-27 03:34:07.361242\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-02-27 03:34:07.362196\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-65\"\n  \"ts\"=\"2026-02-27 03:34:07.362211\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"tls-endpoint-cluster-8p26lt\"\n  \"ts\"=\"2026-02-27 03:34:07.531799\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"tls-endpoint-cluster-8p26lt\" \"deployment\"=\"pooledNonMsiScopedAssignments-oo7pyvaeowk44\"\n  \"ts\"=\"2026-02-27 03:34:07.673589\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-65\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-27 03:34:07.799985\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-65\" \"deployment\"=\"msi-deployment-65\"\n  \"ts\"=\"2026-02-27 03:34:08.041514\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"tls-endpoint-cluster-8p26lt\" \"deployment\"=\"customer-infra-tls-endpoint-hcp-cluster-t8phmj\"\n  \"ts\"=\"2026-02-27 03:34:08.198233\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-02-27 03:34:08.198349\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-02-27 03:34:08.198504\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"tls-endpoint-cluster-8p26lt\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n: 1h29m59.786s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:39\n        In [It] (Node Runtime: 1h29m59.786s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:39\n          At [By Step] creating the node pool (Step Runtime: 43m10.278s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:103\n\n          Spec Goroutine\n          goroutine 112 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x820b858, 0xc000dd0550}, 0x77f1e58?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x824f540, {0x820b858, 0xc000dd0550}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateNodePoolAndWait({0x820b8c8, 0xc000705490}, 0xc0000976f8, {0xc0015469c0, 0x1b}, {0x782be91, 0x18}, {0x77d2568, 0x4}, {0xc0017f1a80, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:718\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateNodePoolFromParam(0xc000deb8c0, {0x7f2a7c514b98, 0xc000a42540}, {0xc0015469c0, 0x1b}, {0x782be91, 0x18}, {{0x77d5c37, 0x6}, {0x782be91, ...}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:384\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func13.2({0x7f2a7c514b98, 0xc000a42540})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:108\n                | nodePoolParams.NodePoolName = customerNodePoolName\n                | \n                \u003e err = tc.CreateNodePoolFromParam(ctx,\n                | \t*resourceGroup.Name,\n                | \tcustomerClusterName,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82210c0?, 0xc000a42540?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 109\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:114 @ 02/27/26 03:34:05.194\n  \"ts\"=\"2026-02-27 03:34:05.194982\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-02-27 03:34:05.195051\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-tls-endpoint-hcp-cluster-t8phmj\" \"resourceGroup\"=\"tls-endpoint-cluster-8p26lt\"\n  \"ts\"=\"2026-02-27 03:34:05.722278\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-tls-endpoint-hcp-cluster-t8phmj\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-02-27 03:34:07.361055\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-81f205190df882aeed9c971ba5a51da44c00eb526b484755681d39236b7f3906.yaml.gz\"\n  \"ts\"=\"2026-02-27 03:34:07.361174\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-81f205190df882aeed9c971ba5a51da44c00eb526b484755681d39236b7f3906.yaml.gz\"\n  \"ts\"=\"2026-02-27 03:34:07.361242\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-02-27 03:34:07.362196\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-65\"\n  \"ts\"=\"2026-02-27 03:34:07.362211\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"tls-endpoint-cluster-8p26lt\"\n  \"ts\"=\"2026-02-27 03:34:07.531799\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"tls-endpoint-cluster-8p26lt\" \"deployment\"=\"pooledNonMsiScopedAssignments-oo7pyvaeowk44\"\n  \"ts\"=\"2026-02-27 03:34:07.673589\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-65\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-27 03:34:07.799985\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-65\" \"deployment\"=\"msi-deployment-65\"\n  \"ts\"=\"2026-02-27 03:34:08.041514\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"tls-endpoint-cluster-8p26lt\" \"deployment\"=\"customer-infra-tls-endpoint-hcp-cluster-t8phmj\"\n  \"ts\"=\"2026-02-27 03:34:08.198233\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-02-27 03:34:08.198349\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-02-27 03:34:08.198504\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"tls-endpoint-cluster-8p26lt\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","d a nodepool with autoscaling enabled (Spec Runtime: 1h29m59.956s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36\n        In [It] (Node Runtime: 1h29m59.956s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36\n          At [By Step] creating the autoscaling nodepool (Step Runtime: 43m12.88s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:95\n\n          Spec Goroutine\n          goroutine 115 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x820b858, 0xc000b222d0}, 0x77f1e58?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x824f540, {0x820b858, 0xc000b222d0}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateNodePoolAndWait({0x820b8c8, 0xc000306b60}, 0xc000097788, {0xc000b3c0f0, 0x15}, {0x78182bc, 0x14}, {0x77f1e34, 0xc}, {0xc000809e30, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:718\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateNodePoolFromParam(0xc00087de60, {0x7f084c347788, 0xc00084cd80}, {0xc000b3c0f0, 0x15}, {0x78182bc, 0x14}, {{0x77d5c37, 0x6}, {0x78182bc, ...}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:384\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func23.2({0x7f084c347788, 0xc00084cd80})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:96\n                | \n                | By(\"creating the autoscaling nodepool\")\n                \u003e err = tc.CreateNodePoolFromParam(ctx,\n                | \t*resourceGroup.Name,\n                | \tcustomerClusterName,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82210c0?, 0xc00084cd80?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 112\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102 @ 02/27/26 03:34:06.87\n  \"ts\"=\"2026-02-27 03:34:06.870227\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-02-27 03:34:06.870286\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-autoscale-cluster-vt8brb\" \"resourceGroup\"=\"np-autoscaling-9pd7cm\"\n  \"ts\"=\"2026-02-27 03:34:07.029064\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-autoscale-cluster-vt8brb\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-02-27 03:34:08.813086\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n  \"ts\"=\"2026-02-27 03:34:08.813201\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n  \"ts\"=\"2026-02-27 03:34:08.813273\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-02-27 03:34:08.814251\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-10\"\n  \"ts\"=\"2026-02-27 03:34:08.814264\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"np-autoscaling-9pd7cm\"\n  \"ts\"=\"2026-02-27 03:34:09.189947\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-9pd7cm\" \"deployment\"=\"pooledNonMsiScopedAssignments-ssrwqrnhyyabs\"\n  \"ts\"=\"2026-02-27 03:34:09.337198\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-10\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-27 03:34:09.337744\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-9pd7cm\" \"deployment\"=\"customer-infra-np-autoscale-cluster-vt8brb\"\n  \"ts\"=\"2026-02-27 03:34:09.474232\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-10\" \"deployment\"=\"msi-deployment-10\"\n  \"ts\"=\"2026-02-27 03:34:09.613226\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-02-27 03:34:09.613375\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-02-27 03:34:09.613494\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"np-autoscaling-9pd7cm\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nd a nodepool with autoscaling enabled (Spec Runtime: 1h29m59.956s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36\n        In [It] (Node Runtime: 1h29m59.956s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36\n          At [By Step] creating the autoscaling nodepool (Step Runtime: 43m12.88s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:95\n\n          Spec Goroutine\n          goroutine 115 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x820b858, 0xc000b222d0}, 0x77f1e58?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x824f540, {0x820b858, 0xc000b222d0}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateNodePoolAndWait({0x820b8c8, 0xc000306b60}, 0xc000097788, {0xc000b3c0f0, 0x15}, {0x78182bc, 0x14}, {0x77f1e34, 0xc}, {0xc000809e30, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:718\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateNodePoolFromParam(0xc00087de60, {0x7f084c347788, 0xc00084cd80}, {0xc000b3c0f0, 0x15}, {0x78182bc, 0x14}, {{0x77d5c37, 0x6}, {0x78182bc, ...}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:384\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func23.2({0x7f084c347788, 0xc00084cd80})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:96\n                | \n                | By(\"creating the autoscaling nodepool\")\n                \u003e err = tc.CreateNodePoolFromParam(ctx,\n                | \t*resourceGroup.Name,\n                | \tcustomerClusterName,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82210c0?, 0xc00084cd80?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 112\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102 @ 02/27/26 03:34:06.87\n  \"ts\"=\"2026-02-27 03:34:06.870227\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-02-27 03:34:06.870286\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-autoscale-cluster-vt8brb\" \"resourceGroup\"=\"np-autoscaling-9pd7cm\"\n  \"ts\"=\"2026-02-27 03:34:07.029064\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-autoscale-cluster-vt8brb\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-02-27 03:34:08.813086\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n  \"ts\"=\"2026-02-27 03:34:08.813201\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n  \"ts\"=\"2026-02-27 03:34:08.813273\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-02-27 03:34:08.814251\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-10\"\n  \"ts\"=\"2026-02-27 03:34:08.814264\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"np-autoscaling-9pd7cm\"\n  \"ts\"=\"2026-02-27 03:34:09.189947\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-9pd7cm\" \"deployment\"=\"pooledNonMsiScopedAssignments-ssrwqrnhyyabs\"\n  \"ts\"=\"2026-02-27 03:34:09.337198\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-10\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-27 03:34:09.337744\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-9pd7cm\" \"deployment\"=\"customer-infra-np-autoscale-cluster-vt8brb\"\n  \"ts\"=\"2026-02-27 03:34:09.474232\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-10\" \"deployment\"=\"msi-deployment-10\"\n  \"ts\"=\"2026-02-27 03:34:09.613226\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-02-27 03:34:09.613375\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-02-27 03:34:09.613494\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"np-autoscaling-9pd7cm\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027202235628261376","occurred_at":"2026-02-27T02:00:51Z","signature_id":"2d7e32582cdb1b1765f3322fc2776786bacdad60817ff21ebc888d518679ce3e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027202235628261376","occurred_at":"2026-02-27T02:00:51Z","signature_id":"4a46b042f4151780516da040d8a9e7d341d21a7c806ef8345873baaf7782c13b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f1386f529db6e94f57caba5baeea25a7ed546b620757afe7e7438f6ec564ee32","failure_pattern":"CreateOrUpdateExternalAuthAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for external auth \"ea-list\" in resourcegroup=\"ea-list-dwdpnj\" for cluster=\"ea-list677v96\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-list677v96 in resource group ea-list-dwdpnj, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-02-21..2026-02-27 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:118]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013c5b30\u003e: \n    failed waiting for external auth \"ea-list\" in resourcegroup=\"ea-list-dwdpnj\" for cluster=\"ea-list677v96\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-list677v96 in resource group ea-list-dwdpnj, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"ea-list\\\" in resourcegroup=\\\"ea-list-dwdpnj\\\" for cluster=\\\"ea-list677v96\\\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-list677v96 in resource group ea-list-dwdpnj, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000718220\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-list677v96 in resource group ea-list-dwdpnj\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:118]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013c5b30\u003e: \n    failed waiting for external auth \"ea-list\" in resourcegroup=\"ea-list-dwdpnj\" for cluster=\"ea-list677v96\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-list677v96 in resource group ea-list-dwdpnj, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"ea-list\\\" in resourcegroup=\\\"ea-list-dwdpnj\\\" for cluster=\\\"ea-list677v96\\\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-list677v96 in resource group ea-list-dwdpnj, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000718220\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-list677v96 in resource group ea-list-dwdpnj\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027202235628261376","occurred_at":"2026-02-27T02:00:51Z","signature_id":"10a285abc8fefb8fd35edb940c2261a24a83f7f594dea4d95377698e412f43cd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-02-21..2026-02-27 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:47]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:47]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027202235628261376","occurred_at":"2026-02-27T02:00:51Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}