{"meta":{"start_date":"2026-02-28","end_date":"2026-02-28","start_at":"2026-02-28T00:00:00Z","end_at":"2026-03-01T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T21:35:52Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":1,"failed_runs":1,"raw_occurrences":12,"matched_occurrences":12,"runs_affected":1},"rows":[{"environment":"prod","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-rzlb97, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-rzlb97\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":7,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":7,"after_last_push_count":7,"trend_counts":[0,0,0,0,0,0,7],"trend_range":"2026-02-22..2026-02-28 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:242]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001188e70\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-z47mk2, error: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-z47mk2\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-z47mk2, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-z47mk2\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00118bd60\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-z47mk2\",\n            },\n            \u003c*fmt.wrapError | 0xc0015b8820\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-z47mk2\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:242]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001188e70\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-z47mk2, error: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-z47mk2\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-z47mk2, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-z47mk2\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00118bd60\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group e2e-cidr-connectivity-z47mk2\",\n            },\n            \u003c*fmt.wrapError | 0xc0015b8820\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-z47mk2\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:124]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011e9170\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-6hwj2w, error: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-6hwj2w\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-6hwj2w, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-6hwj2w\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0006e0930\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-6hwj2w\",\n            },\n            \u003c*fmt.wrapError | 0xc0008f6b20\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-6hwj2w\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:124]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011e9170\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-6hwj2w, error: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-6hwj2w\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-6hwj2w, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-6hwj2w\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0006e0930\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-6hwj2w\",\n            },\n            \u003c*fmt.wrapError | 0xc0008f6b20\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-6hwj2w\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:94]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000baea50\u003e: \n    failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-4g2rmk, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-4g2rmk\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-4g2rmk, error: failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-4g2rmk\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000b4eb90\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-4g2rmk\",\n            },\n            \u003c*fmt.wrapError | 0xc00119c2a0\u003e{\n                msg: \"failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-4g2rmk\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:94]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000baea50\u003e: \n    failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-4g2rmk, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-4g2rmk\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-4g2rmk, error: failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-4g2rmk\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000b4eb90\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-4g2rmk\",\n            },\n            \u003c*fmt.wrapError | 0xc00119c2a0\u003e{\n                msg: \"failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-4g2rmk\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027564627688165376","occurred_at":"2026-02-28T02:00:52Z","signature_id":"321b9d7ecdb9f8c8f8f9801172ea1457965b3053191d0f9d460d5ff512f794a2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027564627688165376","occurred_at":"2026-02-28T02:00:52Z","signature_id":"59f326b720281593e72f0fdec29f3ebe6366aecd224e79e016c5c0a19ea026b8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027564627688165376","occurred_at":"2026-02-28T02:00:52Z","signature_id":"6a152062aa58182f5d1e64979999ec71101c12b48d64f90537890f28647d2709","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027564627688165376","occurred_at":"2026-02-28T02:00:52Z","signature_id":"9dd35b98a492217681370474bb165425a78d7b4ec4ae720ad461976cca255166","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027564627688165376","occurred_at":"2026-02-28T02:00:52Z","signature_id":"c2710caf6c53c9655c883d0f3c6c5856ab187057e4769f7d4755d79ba6b95547","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027564627688165376","occurred_at":"2026-02-28T02:00:52Z","signature_id":"db76c9eebdc326949a940d0d95ec664c09c175467362f753aaba0b46879230ae","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027564627688165376","occurred_at":"2026-02-28T02:00:52Z","signature_id":"e1faf50e90a689822303bf5999c4643c6dd0c86b8a51339f906a2dac5e3e8470","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":3,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,2,3],"trend_range":"2026-02-22..2026-02-28 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["eanup and reporting nodes but will skip all remaining specs.  Interrupt again to skip cleanup.\n    Here's a current progress report:\n      Customer should be able to create an HCP cluster with back-level version 4.19 (Spec Runtime: 1h29m59.881s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:48\n        In [It] (Node Runtime: 1h29m59.881s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:48\n          At [By Step] creating node pool version 4.19.7 and verifying a simple web app can run (Step Runtime: 44m14.857s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:174\n\n          Spec Goroutine\n          goroutine 46 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x820bba8, 0xc0005f2af0}, 0x77f1f7b?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x824f780, {0x820bba8, 0xc0005f2af0}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateNodePoolAndWait({0x7fb640315f60, 0xc000e897a0}, 0xc000095d10, {0xc0013f40e0, 0x1e}, {0xc0013da330, 0x10}, {0xc000a3e680, 0xd}, {0xc000e3a4e0, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:757\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func15.1({0x7fb640315f60, 0xc000e897a0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:182\n                | Expect(err).NotTo(HaveOccurred())\n                | nodePoolClient := tc.Get20240610ClientFactoryOrDie(ctx).NewNodePoolsClient()\n                \u003e _, err = framework.CreateNodePoolAndWait(ctx,\n                | \tnodePoolClient,\n                | \t*resourceGroup.Name,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213a0?, 0xc000e897a0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 87\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:190 @ 02/28/26 03:33:42.095\n  \"ts\"=\"2026-02-28 03:33:42.095647\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-02-28 03:33:42.095721\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-ver-4-19-sqgvfd\" \"resourceGroup\"=\"rg-cluster-back-version-phh9hp\"\n  \"ts\"=\"2026-02-28 03:33:42.266832\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-ver-4-19-rhxxsh\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-02-28 03:33:44.552401\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b47f269c3c8e3f0a236ed67162dd7947c6f274b6961894c6582aefea4761a7f3.yaml.gz\"\n  \"ts\"=\"2026-02-28 03:33:44.552493\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b47f269c3c8e3f0a236ed67162dd7947c6f274b6961894c6582aefea4761a7f3.yaml.gz\"\n  \"ts\"=\"2026-02-28 03:33:44.552551\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-02-28 03:33:44.553626\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-75\"\n  \"ts\"=\"2026-02-28 03:33:44.553643\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-cluster-back-version-phh9hp\"\n  \"ts\"=\"2026-02-28 03:33:44.694802\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-75\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-28 03:33:44.701210\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-cluster-back-version-phh9hp\" \"deployment\"=\"pooledNonMsiScopedAssignments\"\n  \"ts\"=\"2026-02-28 03:33:44.845805\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-cluster-back-version-phh9hp\" \"deployment\"=\"customer-infra-cluster-ver-4-19-sqgvfd\"\n  \"ts\"=\"2026-02-28 03:33:44.893842\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-75\" \"deployment\"=\"msi-deployment-75\"\n  \"ts\"=\"2026-02-28 03:33:45.046661\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-02-28 03:33:45.046802\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-02-28 03:33:45.046928\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-cluster-back-version-phh9hp\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\neanup and reporting nodes but will skip all remaining specs.  Interrupt again to skip cleanup.\n    Here's a current progress report:\n      Customer should be able to create an HCP cluster with back-level version 4.19 (Spec Runtime: 1h29m59.881s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:48\n        In [It] (Node Runtime: 1h29m59.881s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:48\n          At [By Step] creating node pool version 4.19.7 and verifying a simple web app can run (Step Runtime: 44m14.857s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:174\n\n          Spec Goroutine\n          goroutine 46 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x820bba8, 0xc0005f2af0}, 0x77f1f7b?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x824f780, {0x820bba8, 0xc0005f2af0}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateNodePoolAndWait({0x7fb640315f60, 0xc000e897a0}, 0xc000095d10, {0xc0013f40e0, 0x1e}, {0xc0013da330, 0x10}, {0xc000a3e680, 0xd}, {0xc000e3a4e0, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:757\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func15.1({0x7fb640315f60, 0xc000e897a0})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:182\n                | Expect(err).NotTo(HaveOccurred())\n                | nodePoolClient := tc.Get20240610ClientFactoryOrDie(ctx).NewNodePoolsClient()\n                \u003e _, err = framework.CreateNodePoolAndWait(ctx,\n                | \tnodePoolClient,\n                | \t*resourceGroup.Name,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213a0?, 0xc000e897a0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 87\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:190 @ 02/28/26 03:33:42.095\n  \"ts\"=\"2026-02-28 03:33:42.095647\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-02-28 03:33:42.095721\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-ver-4-19-sqgvfd\" \"resourceGroup\"=\"rg-cluster-back-version-phh9hp\"\n  \"ts\"=\"2026-02-28 03:33:42.266832\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-ver-4-19-rhxxsh\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-02-28 03:33:44.552401\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b47f269c3c8e3f0a236ed67162dd7947c6f274b6961894c6582aefea4761a7f3.yaml.gz\"\n  \"ts\"=\"2026-02-28 03:33:44.552493\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b47f269c3c8e3f0a236ed67162dd7947c6f274b6961894c6582aefea4761a7f3.yaml.gz\"\n  \"ts\"=\"2026-02-28 03:33:44.552551\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-02-28 03:33:44.553626\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-75\"\n  \"ts\"=\"2026-02-28 03:33:44.553643\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-cluster-back-version-phh9hp\"\n  \"ts\"=\"2026-02-28 03:33:44.694802\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-75\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-28 03:33:44.701210\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-cluster-back-version-phh9hp\" \"deployment\"=\"pooledNonMsiScopedAssignments\"\n  \"ts\"=\"2026-02-28 03:33:44.845805\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-cluster-back-version-phh9hp\" \"deployment\"=\"customer-infra-cluster-ver-4-19-sqgvfd\"\n  \"ts\"=\"2026-02-28 03:33:44.893842\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-75\" \"deployment\"=\"msi-deployment-75\"\n  \"ts\"=\"2026-02-28 03:33:45.046661\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-02-28 03:33:45.046802\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-02-28 03:33:45.046928\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-cluster-back-version-phh9hp\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","0x501a44d\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213a0?, 0xc0008d6d20?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e fp=0xc000095f88 sp=0xc000095f58 pc=0x2eed25e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000095fe0 sp=0xc000095f88 pc=0x2f0197b\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000095fe8 sp=0xc000095fe0 pc=0x2ac5921\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 100\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\ngoroutine 110 gp=0xc0014ae1c0 m=nil [IO wait]:\nruntime.gopark(0xc0015de140?, 0x6c6b8100?, 0x0?, 0x97?, 0xb?)\n\t/usr/local/go/src/runtime/proc.go:460 +0xce fp=0xc0000c9720 sp=0xc0000c9700 pc=0x2abd28e\nruntime.netpollblock(0x2b26d58?, 0x2a49046?, 0x0?)\n\t/usr/local/go/src/runtime/netpoll.go:575 +0xf7 fp=0xc0000c9758 sp=0xc0000c9720 pc=0x2a7ce97\ninternal/poll.runtime_pollWait(0x7f496e014600, 0x72)\n\t/usr/local/go/src/runtime/netpoll.go:351 +0x85 fp=0xc0000c9778 sp=0xc0000c9758 pc=0x2abbb25\ninternal/poll.(*pollDesc).wait(0xc000175500?, 0xc000e8c000?, 0x0)\n\t/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc0000c97a0 sp=0xc0000c9778 pc=0x2b49a87\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/local/go/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc000175500, {0xc000e8c000, 0x3500, 0x3500})\n\t/usr/local/go/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc0000c9838 sp=0xc0000c97a0 pc=0x2b4aeb9\nnet.(*netFD).Read(0xc000175500, {0xc000e8c000?, 0xc000e8c2c1?, 0x5?})\n\t/usr/local/go/src/net/fd_posix.go:68 +0x25 fp=0xc0000c9880 sp=0xc0000c9838 pc=0x2cdf185\nnet.(*conn).Read(0xc00012a2c8, {0xc000e8c000?, 0x7f496c6b6c88?, 0x7f49b4bfb5c0?})\n\t/usr/local/go/src/net/net.go:196 +0x45 fp=0xc0000c98c8 sp=0xc0000c9880 pc=0x2cf1225\ncrypto/tls.(*atLeastReader).Read(0xc001822558, {0xc000e8c000?, 0x323a?, 0x2aba085?})\n\t/usr/local/go/src/crypto/tls/conn.go:819 +0x3b fp=0xc0000c9910 sp=0xc0000c98c8 pc=0x2d5cadb\nbytes.(*Buffer).ReadFrom(0xc0008b82a8, {0x81730a0, 0xc001822558})\n\t/usr/local/go/src/bytes/buffer.go:217 +0x98 fp=0xc0000c9968 sp=0xc0000c9910 pc=0x2b87a38\ncrypto/tls.(*Conn).readFromUntil(0xc0008b8008, {0x8173aa0, 0xc00012a2c8}, 0xc0000c99d0?)\n\t/usr/local/go/src/crypto/tls/conn.go:841 +0xde fp=0xc0000c99a0 sp=0xc0000c9968 pc=0x2d5ccbe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc0008b8008, 0x0)\n\t/usr/local/go/src/crypto/tls/conn.go:630 +0x3db fp=0xc0000c9c18 sp=0xc0000c99a0 pc=0x2d59d3b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/local/go/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc0008b8008, {0xc0015fd000, 0x1000, 0x319b220?})\n\t/usr/local/go/src/crypto/tls/conn.go:1397 +0x145 fp=0xc0000c9c80 sp=0xc0000c9c18 pc=0x2d60625\nbufio.(*Reader).Read(0xc0015bccc0, {0xc0015da204, 0x9, 0x31a936e?})\n\t/usr/local/go/src/bufio/bufio.go:245 +0x197 fp=0xc0000c9cb8 sp=0xc0000c9c80 pc=0x2c22617\nio.ReadAtLeast({0x8172680, 0xc0015bccc0}, {0xc0015da204, 0x9, 0x9}, 0x9)\n\t/usr/local/go/src/io/io.go:335 +0x8e fp=0xc0000c9d00 sp=0xc0000c9cb8 pc=0x2b17aee\nio.ReadFull(...)\n\t/usr/local/go/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc0015da204, 0x9, 0xc00000022b?}, {0x8172680?, 0xc0015bccc0?})\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/frame.go:242 +0x65 fp=0xc0000c9d50 sp=0xc0000c9d00 pc=0x31929c5\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc0015da1c0)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/frame.go:505 +0x6b fp=0xc0000c9dd8 sp=0xc0000c9d50 pc=0x319358b\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc0015da1c0)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/frame.go:564 +0x18 fp=0xc0000c9e18 sp=0xc0000c9dd8 pc=0x3193cf8\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc0000c9fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:2258 +0xca fp=0xc0000c9f60 sp=0xc0000c9e18 pc=0x31a62aa\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc0015dc380)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:2127 +0x52 fp=0xc0000c9fc8 sp=0xc0000c9f60 pc=0x31a5632\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:880 +0x25 fp=0xc0000c9fe0 sp=0xc0000c9fc8 pc=0x319edc5\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000c9fe8 sp=0xc0000c9fe0 pc=0x2ac5921\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 109\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:880 +0xda5\n\nrax    0xca\nrbx    0x0\nrcx    0x2ac7743\nrdx    0x0\nrdi    0xb2ef450\nrsi    0x80\nrbp    0x7fff363f7648\nrsp    0x7fff363f7600\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x86\nr13    0x1\nr14    0xb2ed360\nr15    0x5\nrip    0x2ac7741\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\n0x501a44d\ngithub.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213a0?, 0xc0008d6d20?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465 +0x3e fp=0xc000095f88 sp=0xc000095f58 pc=0x2eed25e\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000095fe0 sp=0xc000095f88 pc=0x2f0197b\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000095fe8 sp=0xc000095fe0 pc=0x2ac5921\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 100\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\ngoroutine 110 gp=0xc0014ae1c0 m=nil [IO wait]:\nruntime.gopark(0xc0015de140?, 0x6c6b8100?, 0x0?, 0x97?, 0xb?)\n\t/usr/local/go/src/runtime/proc.go:460 +0xce fp=0xc0000c9720 sp=0xc0000c9700 pc=0x2abd28e\nruntime.netpollblock(0x2b26d58?, 0x2a49046?, 0x0?)\n\t/usr/local/go/src/runtime/netpoll.go:575 +0xf7 fp=0xc0000c9758 sp=0xc0000c9720 pc=0x2a7ce97\ninternal/poll.runtime_pollWait(0x7f496e014600, 0x72)\n\t/usr/local/go/src/runtime/netpoll.go:351 +0x85 fp=0xc0000c9778 sp=0xc0000c9758 pc=0x2abbb25\ninternal/poll.(*pollDesc).wait(0xc000175500?, 0xc000e8c000?, 0x0)\n\t/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc0000c97a0 sp=0xc0000c9778 pc=0x2b49a87\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/local/go/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc000175500, {0xc000e8c000, 0x3500, 0x3500})\n\t/usr/local/go/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc0000c9838 sp=0xc0000c97a0 pc=0x2b4aeb9\nnet.(*netFD).Read(0xc000175500, {0xc000e8c000?, 0xc000e8c2c1?, 0x5?})\n\t/usr/local/go/src/net/fd_posix.go:68 +0x25 fp=0xc0000c9880 sp=0xc0000c9838 pc=0x2cdf185\nnet.(*conn).Read(0xc00012a2c8, {0xc000e8c000?, 0x7f496c6b6c88?, 0x7f49b4bfb5c0?})\n\t/usr/local/go/src/net/net.go:196 +0x45 fp=0xc0000c98c8 sp=0xc0000c9880 pc=0x2cf1225\ncrypto/tls.(*atLeastReader).Read(0xc001822558, {0xc000e8c000?, 0x323a?, 0x2aba085?})\n\t/usr/local/go/src/crypto/tls/conn.go:819 +0x3b fp=0xc0000c9910 sp=0xc0000c98c8 pc=0x2d5cadb\nbytes.(*Buffer).ReadFrom(0xc0008b82a8, {0x81730a0, 0xc001822558})\n\t/usr/local/go/src/bytes/buffer.go:217 +0x98 fp=0xc0000c9968 sp=0xc0000c9910 pc=0x2b87a38\ncrypto/tls.(*Conn).readFromUntil(0xc0008b8008, {0x8173aa0, 0xc00012a2c8}, 0xc0000c99d0?)\n\t/usr/local/go/src/crypto/tls/conn.go:841 +0xde fp=0xc0000c99a0 sp=0xc0000c9968 pc=0x2d5ccbe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc0008b8008, 0x0)\n\t/usr/local/go/src/crypto/tls/conn.go:630 +0x3db fp=0xc0000c9c18 sp=0xc0000c99a0 pc=0x2d59d3b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/local/go/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc0008b8008, {0xc0015fd000, 0x1000, 0x319b220?})\n\t/usr/local/go/src/crypto/tls/conn.go:1397 +0x145 fp=0xc0000c9c80 sp=0xc0000c9c18 pc=0x2d60625\nbufio.(*Reader).Read(0xc0015bccc0, {0xc0015da204, 0x9, 0x31a936e?})\n\t/usr/local/go/src/bufio/bufio.go:245 +0x197 fp=0xc0000c9cb8 sp=0xc0000c9c80 pc=0x2c22617\nio.ReadAtLeast({0x8172680, 0xc0015bccc0}, {0xc0015da204, 0x9, 0x9}, 0x9)\n\t/usr/local/go/src/io/io.go:335 +0x8e fp=0xc0000c9d00 sp=0xc0000c9cb8 pc=0x2b17aee\nio.ReadFull(...)\n\t/usr/local/go/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc0015da204, 0x9, 0xc00000022b?}, {0x8172680?, 0xc0015bccc0?})\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/frame.go:242 +0x65 fp=0xc0000c9d50 sp=0xc0000c9d00 pc=0x31929c5\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc0015da1c0)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/frame.go:505 +0x6b fp=0xc0000c9dd8 sp=0xc0000c9d50 pc=0x319358b\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc0015da1c0)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/frame.go:564 +0x18 fp=0xc0000c9e18 sp=0xc0000c9dd8 pc=0x3193cf8\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc0000c9fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:2258 +0xca fp=0xc0000c9f60 sp=0xc0000c9e18 pc=0x31a62aa\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc0015dc380)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:2127 +0x52 fp=0xc0000c9fc8 sp=0xc0000c9f60 pc=0x31a5632\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:880 +0x25 fp=0xc0000c9fe0 sp=0xc0000c9fc8 pc=0x319edc5\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc0000c9fe8 sp=0xc0000c9fe0 pc=0x2ac5921\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 109\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:880 +0xda5\n\nrax    0xca\nrbx    0x0\nrcx    0x2ac7743\nrdx    0x0\nrdi    0xb2ef450\nrsi    0x80\nrbp    0x7fff363f7648\nrsp    0x7fff363f7600\nr8     0x0\nr9     0x0\nr10    0x0\nr11    0x286\nr12    0x86\nr13    0x1\nr14    0xb2ed360\nr15    0x5\nrip    0x2ac7741\nrflags 0x286\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command","Customer should be able to create an HCP cluster (Spec Runtime: 1h29m59.887s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:38\n        In [It] (Node Runtime: 1h29m59.887s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:38\n          At [By Step] creating the node pool (Step Runtime: 27m42.566s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:106\n\n          Spec Goroutine\n          goroutine 104 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x820bb38, 0xc000ab9b80}, 0x77f1f7b?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x824f780, {0x820bb38, 0xc000ab9b80}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateNodePoolAndWait({0x820bba8, 0xc0008813b0}, 0xc000095790, {0xc000a767b0, 0x14}, {0x7808b9f, 0x11}, {0x77d2688, 0x4}, {0xc00033a140, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:757\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateNodePoolFromParam(0xc000ecad80, {0x7fdde151a260, 0xc000a30b10}, {0xc000a767b0, 0x14}, {0x7808b9f, 0x11}, {{0x77d5d51, 0x6}, {0x7808b9f, ...}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:384\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func17.2({0x7fdde151a260, 0xc000a30b10})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:112\n                | nodePoolParams.Replicas = int32(2)\n                | \n                \u003e err = tc.CreateNodePoolFromParam(ctx,\n                | \t*resourceGroup.Name,\n                | \tcustomerClusterName,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213a0?, 0xc000a30b10?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 101\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:118 @ 02/28/26 03:33:42.095\n  \"ts\"=\"2026-02-28 03:33:42.095738\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-02-28 03:33:42.095808\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-2g8z72\" \"resourceGroup\"=\"basic-cluster-gwc8xh\"\n  \"ts\"=\"2026-02-28 03:33:42.636827\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-2g8z72\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-02-28 03:33:44.365594\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-8d7dc92573349c6d0052ecc8bc03042b1bfdfade27aae309024a834a40d01d5c.yaml.gz\"\n  \"ts\"=\"2026-02-28 03:33:44.365704\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-8d7dc92573349c6d0052ecc8bc03042b1bfdfade27aae309024a834a40d01d5c.yaml.gz\"\n  \"ts\"=\"2026-02-28 03:33:44.365770\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-02-28 03:33:44.366773\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-41\"\n  \"ts\"=\"2026-02-28 03:33:44.366788\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"basic-cluster-gwc8xh\"\n  \"ts\"=\"2026-02-28 03:33:44.532786\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-41\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-28 03:33:44.547739\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"basic-cluster-gwc8xh\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-02-28 03:33:44.674430\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-41\" \"deployment\"=\"msi-deployment-41\"\n  \"ts\"=\"2026-02-28 03:33:44.681140\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"basic-cluster-gwc8xh\" \"deployment\"=\"customer-infra-basic-hcp-cluster-2g8z72\"\n  \"ts\"=\"2026-02-28 03:33:45.089935\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-02-28 03:33:45.090052\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-02-28 03:33:45.090215\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"basic-cluster-gwc8xh\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nCustomer should be able to create an HCP cluster (Spec Runtime: 1h29m59.887s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:38\n        In [It] (Node Runtime: 1h29m59.887s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:38\n          At [By Step] creating the node pool (Step Runtime: 27m42.566s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:106\n\n          Spec Goroutine\n          goroutine 104 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x820bb38, 0xc000ab9b80}, 0x77f1f7b?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x824f780, {0x820bb38, 0xc000ab9b80}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateNodePoolAndWait({0x820bba8, 0xc0008813b0}, 0xc000095790, {0xc000a767b0, 0x14}, {0x7808b9f, 0x11}, {0x77d2688, 0x4}, {0xc00033a140, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:757\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateNodePoolFromParam(0xc000ecad80, {0x7fdde151a260, 0xc000a30b10}, {0xc000a767b0, 0x14}, {0x7808b9f, 0x11}, {{0x77d5d51, 0x6}, {0x7808b9f, ...}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:384\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func17.2({0x7fdde151a260, 0xc000a30b10})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:112\n                | nodePoolParams.Replicas = int32(2)\n                | \n                \u003e err = tc.CreateNodePoolFromParam(ctx,\n                | \t*resourceGroup.Name,\n                | \tcustomerClusterName,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213a0?, 0xc000a30b10?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 101\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:118 @ 02/28/26 03:33:42.095\n  \"ts\"=\"2026-02-28 03:33:42.095738\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-02-28 03:33:42.095808\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-2g8z72\" \"resourceGroup\"=\"basic-cluster-gwc8xh\"\n  \"ts\"=\"2026-02-28 03:33:42.636827\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-2g8z72\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-02-28 03:33:44.365594\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-8d7dc92573349c6d0052ecc8bc03042b1bfdfade27aae309024a834a40d01d5c.yaml.gz\"\n  \"ts\"=\"2026-02-28 03:33:44.365704\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-8d7dc92573349c6d0052ecc8bc03042b1bfdfade27aae309024a834a40d01d5c.yaml.gz\"\n  \"ts\"=\"2026-02-28 03:33:44.365770\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-02-28 03:33:44.366773\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-41\"\n  \"ts\"=\"2026-02-28 03:33:44.366788\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"basic-cluster-gwc8xh\"\n  \"ts\"=\"2026-02-28 03:33:44.532786\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-41\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-02-28 03:33:44.547739\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"basic-cluster-gwc8xh\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-02-28 03:33:44.674430\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-41\" \"deployment\"=\"msi-deployment-41\"\n  \"ts\"=\"2026-02-28 03:33:44.681140\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"basic-cluster-gwc8xh\" \"deployment\"=\"customer-infra-basic-hcp-cluster-2g8z72\"\n  \"ts\"=\"2026-02-28 03:33:45.089935\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-02-28 03:33:45.090052\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-02-28 03:33:45.090215\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"basic-cluster-gwc8xh\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027564627688165376","occurred_at":"2026-02-28T02:00:52Z","signature_id":"518660464b2a8cfb3680304c0cc7b960cfd647f7410dedaf2d2d51b0ed149a94","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027564627688165376","occurred_at":"2026-02-28T02:00:52Z","signature_id":"7ea01132226a033bf46db5cab003e57dab4e5a88f35103643099cdbc9482c697","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027564627688165376","occurred_at":"2026-02-28T02:00:52Z","signature_id":"86342e3cc1af52f634b8d7645efcccd6676dcbdbb5aea7cd5dcd7c7f5650c28a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f1386f529db6e94f57caba5baeea25a7ed546b620757afe7e7438f6ec564ee32","failure_pattern":"CreateOrUpdateExternalAuthAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for external auth \"ea-list\" in resourcegroup=\"ea-list-lfg8pd\" for cluster=\"ea-list45lzgb\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-list45lzgb in resource group ea-list-lfg8pd, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-02-22..2026-02-28 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:118]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014545a0\u003e: \n    failed waiting for external auth \"ea-list\" in resourcegroup=\"ea-list-lfg8pd\" for cluster=\"ea-list45lzgb\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-list45lzgb in resource group ea-list-lfg8pd, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"ea-list\\\" in resourcegroup=\\\"ea-list-lfg8pd\\\" for cluster=\\\"ea-list45lzgb\\\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-list45lzgb in resource group ea-list-lfg8pd, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000954420\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-list45lzgb in resource group ea-list-lfg8pd\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:118]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014545a0\u003e: \n    failed waiting for external auth \"ea-list\" in resourcegroup=\"ea-list-lfg8pd\" for cluster=\"ea-list45lzgb\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-list45lzgb in resource group ea-list-lfg8pd, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"ea-list\\\" in resourcegroup=\\\"ea-list-lfg8pd\\\" for cluster=\\\"ea-list45lzgb\\\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-list45lzgb in resource group ea-list-lfg8pd, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000954420\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-list45lzgb in resource group ea-list-lfg8pd\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027564627688165376","occurred_at":"2026-02-28T02:00:52Z","signature_id":"9c087c44e154d0b66afa304264d36f9d1fd3ac75f4f6a60fc803e1285a899e36","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"73e00d260dba8489c1bafd3c8ac83fb533ab4decc2607106f0798a4553356558","failure_pattern":"UpdateHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-pczlh6\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster for cluster patch-tags-cluster in resource group patch-tags-pczlh6, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-02-22..2026-02-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:192]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000cec090\u003e: \n    failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-pczlh6\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster for cluster patch-tags-cluster in resource group patch-tags-pczlh6, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-pczlh6\\\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster for cluster patch-tags-cluster in resource group patch-tags-pczlh6, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00027e1b0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during UpdateHCPCluster for cluster patch-tags-cluster in resource group patch-tags-pczlh6\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:192]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000cec090\u003e: \n    failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-pczlh6\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster for cluster patch-tags-cluster in resource group patch-tags-pczlh6, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-pczlh6\\\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster for cluster patch-tags-cluster in resource group patch-tags-pczlh6, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00027e1b0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during UpdateHCPCluster for cluster patch-tags-cluster in resource group patch-tags-pczlh6\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027564627688165376","occurred_at":"2026-02-28T02:00:52Z","signature_id":"34e312e749ac38d0a4fb80a6f128a7d80edb38c489b530a024f2474b748c5ea6","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}