{"meta":{"start_date":"2026-03-01","end_date":"2026-03-01","start_at":"2026-03-01T00:00:00Z","end_at":"2026-03-02T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T20:36:42Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":1,"failed_runs":1,"raw_occurrences":13,"matched_occurrences":13,"runs_affected":1},"rows":[{"environment":"prod","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-wvfnxv, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-wvfnxv\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":6,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":6,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,0,6],"trend_range":"2026-02-23..2026-03-01 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011cc060\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-b4m6br, error: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-b4m6br\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-b4m6br, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-b4m6br\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000b8c910\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-b4m6br\",\n            },\n            \u003c*fmt.wrapError | 0xc00080a6e0\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-b4m6br\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011cc060\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-b4m6br, error: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-b4m6br\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-b4m6br, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-b4m6br\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000b8c910\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-b4m6br\",\n            },\n            \u003c*fmt.wrapError | 0xc00080a6e0\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-b4m6br\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:114]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000dd08a0\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group tls-endpoint-cluster-5694vf, error: failed waiting for nodepool=\"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-5694vf\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group tls-endpoint-cluster-5694vf, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-5694vf\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0009e2330\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group tls-endpoint-cluster-5694vf\",\n            },\n            \u003c*fmt.wrapError | 0xc000e04740\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-5694vf\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:114]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000dd08a0\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group tls-endpoint-cluster-5694vf, error: failed waiting for nodepool=\"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-5694vf\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group tls-endpoint-cluster-5694vf, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-5694vf\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0009e2330\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group tls-endpoint-cluster-5694vf\",\n            },\n            \u003c*fmt.wrapError | 0xc000e04740\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-5694vf\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:120]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000cd2f00\u003e: \n    failed to create NodePool ea-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool ea-np-1 in resource group external-auth-cluster-dsqvqh, error: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-dsqvqh\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool ea-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool ea-np-1 in resource group external-auth-cluster-dsqvqh, error: failed waiting for nodepool=\\\"ea-np-1\\\" for cluster \\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-dsqvqh\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001423500\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool ea-np-1 in resource group external-auth-cluster-dsqvqh\",\n            },\n            \u003c*fmt.wrapError | 0xc000a015a0\u003e{\n                msg: \"failed waiting for nodepool=\\\"ea-np-1\\\" for cluster \\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-dsqvqh\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:120]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000cd2f00\u003e: \n    failed to create NodePool ea-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool ea-np-1 in resource group external-auth-cluster-dsqvqh, error: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-dsqvqh\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool ea-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool ea-np-1 in resource group external-auth-cluster-dsqvqh, error: failed waiting for nodepool=\\\"ea-np-1\\\" for cluster \\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-dsqvqh\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001423500\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool ea-np-1 in resource group external-auth-cluster-dsqvqh\",\n            },\n            \u003c*fmt.wrapError | 0xc000a015a0\u003e{\n                msg: \"failed waiting for nodepool=\\\"ea-np-1\\\" for cluster \\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-dsqvqh\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027927018498166784","occurred_at":"2026-03-01T02:00:52Z","signature_id":"425523d5d1b368da444e5a3134a78a83ebe4b062391b61a3d956df1cc81194f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027927018498166784","occurred_at":"2026-03-01T02:00:52Z","signature_id":"4ad91d2ddad35479bd510d6173f5ea73d95cff7f2826e013f347012b9f1ae8c2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027927018498166784","occurred_at":"2026-03-01T02:00:52Z","signature_id":"5b743d9634853334d8cd99baf69479bef9ac270ea1390918ce423f37225f4e59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027927018498166784","occurred_at":"2026-03-01T02:00:52Z","signature_id":"8ad580bf4150c329932d5d3d0537b4da491608993ca03d38e9a4da586d823deb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027927018498166784","occurred_at":"2026-03-01T02:00:52Z","signature_id":"e00734417be5af2ac3a34de21d7e2c73459626735f1fcd8cd431194bb55c63ba","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027927018498166784","occurred_at":"2026-03-01T02:00:52Z","signature_id":"efa5061af866adb8649f493c692e48ec71ed5c3851ea3660b2b7045554ebe272","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-88jst9\" in resourcegroup=\"admin-credential-lifecycle-test-sbtn87\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-88jst9 in resource group admin-credential-lifecycle-test-sbtn87, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":3,"after_last_push_count":6,"trend_counts":[0,0,0,0,3,0,3],"trend_range":"2026-02-23..2026-03-01 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e5a810\u003e: \n    failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-wwvzjd\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-wwvzjd, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-wwvzjd\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-wwvzjd, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00072e2f0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-wwvzjd\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e5a810\u003e: \n    failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-wwvzjd\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-wwvzjd, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-wwvzjd\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-wwvzjd, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00072e2f0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-wwvzjd\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d1f650\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-88jst9\" in resourcegroup=\"admin-credential-lifecycle-test-sbtn87\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-88jst9 in resource group admin-credential-lifecycle-test-sbtn87, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-88jst9\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-sbtn87\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-88jst9 in resource group admin-credential-lifecycle-test-sbtn87, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000d6b6f0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-88jst9 in resource group admin-credential-lifecycle-test-sbtn87\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000d1f650\u003e: \n    failed waiting for hcpCluster=\"admin-cred-lifecycle-88jst9\" in resourcegroup=\"admin-credential-lifecycle-test-sbtn87\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-88jst9 in resource group admin-credential-lifecycle-test-sbtn87, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-88jst9\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-sbtn87\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-88jst9 in resource group admin-credential-lifecycle-test-sbtn87, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000d6b6f0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-88jst9 in resource group admin-credential-lifecycle-test-sbtn87\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:105]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000efc060\u003e: \n    failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-xfxfjs\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-xfxfjs, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-xfxfjs\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-xfxfjs, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000b21af0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-xfxfjs\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:105]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000efc060\u003e: \n    failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-xfxfjs\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-xfxfjs, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-xfxfjs\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-xfxfjs, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000b21af0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-xfxfjs\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027927018498166784","occurred_at":"2026-03-01T02:00:52Z","signature_id":"8d1a2735323a6bdd13cde282cfa3f6641ee06198faabea53a895e15219198081","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027927018498166784","occurred_at":"2026-03-01T02:00:52Z","signature_id":"9ad8374eb3c07fac57470f3d7e7cb7f91820f1175d344578c23b875462c11191","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027927018498166784","occurred_at":"2026-03-01T02:00:52Z","signature_id":"aa85cac808c0fb6cfdf0e86c77c73b6b7e8c70985d872a4e25a38a3eef2cc773","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":2,"after_last_push_count":7,"trend_counts":[0,0,0,0,2,3,2],"trend_range":"2026-02-23..2026-03-01 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1}],"full_error_samples":["custom autoscaling (Spec Runtime: 1h29m59.892s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:36\n        In [It] (Node Runtime: 1h29m59.892s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:36\n          At [By Step] creating the node pool (Step Runtime: 43m59.075s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:112\n\n          Spec Goroutine\n          goroutine 96 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x820bb58, 0xc000b26190}, 0x77f1f7b?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x824f7a0, {0x820bb58, 0xc000b26190}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateNodePoolAndWait({0x820bbc8, 0xc00072a1c0}, 0xc000095748, {0xc00127de00, 0x1a}, {0x78273c8, 0x17}, {0x77d2688, 0x4}, {0xc00157e570, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:757\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateNodePoolFromParam(0xc0007867e0, {0x7ffbb42ca660, 0xc000975b00}, {0xc00127de00, 0x1a}, {0x78273c8, 0x17}, {{0x77d5d51, 0x6}, {0x78273c8, ...}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:384\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func6.2({0x7ffbb42ca660, 0xc000975b00})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:118\n                | nodePoolParams.Replicas = int32(2)\n                | \n                \u003e err = tc.CreateNodePoolFromParam(ctx,\n                | \t*resourceGroup.Name,\n                | \tcustomerClusterName,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213c0?, 0xc000975b00?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 93\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:124 @ 03/01/26 03:41:53.656\n  \"ts\"=\"2026-03-01 03:41:53.656795\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-01 03:41:53.656865\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-autoscaling-hcp-cluster-pfg4sv\" \"resourceGroup\"=\"autoscaling-cluster-6mbww8\"\n  \"ts\"=\"2026-03-01 03:41:53.882690\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-autoscaling-hcp-cluster-pfg4sv\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-01 03:41:55.474450\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-31a71a35884d87a4a1de36eceeb324011e1c775154472da33ffc40cde64c08cb.yaml.gz\"\n  \"ts\"=\"2026-03-01 03:41:55.474569\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-31a71a35884d87a4a1de36eceeb324011e1c775154472da33ffc40cde64c08cb.yaml.gz\"\n  \"ts\"=\"2026-03-01 03:41:55.474638\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-01 03:41:55.475586\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-148\"\n  \"ts\"=\"2026-03-01 03:41:55.475602\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"autoscaling-cluster-6mbww8\"\n  \"ts\"=\"2026-03-01 03:41:55.655617\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"autoscaling-cluster-6mbww8\" \"deployment\"=\"pooledNonMsiScopedAssignments-e4ixwyxsfjnje\"\n  \"ts\"=\"2026-03-01 03:41:55.814591\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"autoscaling-cluster-6mbww8\" \"deployment\"=\"customer-infra-autoscaling-hcp-cluster-pfg4sv\"\n  \"ts\"=\"2026-03-01 03:41:55.974500\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-148\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-01 03:41:56.147010\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-148\" \"deployment\"=\"msi-deployment-148\"\n  \"ts\"=\"2026-03-01 03:41:56.300640\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-01 03:41:56.300813\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-01 03:41:56.300956\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"autoscaling-cluster-6mbww8\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\ncustom autoscaling (Spec Runtime: 1h29m59.892s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:36\n        In [It] (Node Runtime: 1h29m59.892s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:36\n          At [By Step] creating the node pool (Step Runtime: 43m59.075s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:112\n\n          Spec Goroutine\n          goroutine 96 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x820bb58, 0xc000b26190}, 0x77f1f7b?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x824f7a0, {0x820bb58, 0xc000b26190}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateNodePoolAndWait({0x820bbc8, 0xc00072a1c0}, 0xc000095748, {0xc00127de00, 0x1a}, {0x78273c8, 0x17}, {0x77d2688, 0x4}, {0xc00157e570, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:757\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateNodePoolFromParam(0xc0007867e0, {0x7ffbb42ca660, 0xc000975b00}, {0xc00127de00, 0x1a}, {0x78273c8, 0x17}, {{0x77d5d51, 0x6}, {0x78273c8, ...}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:384\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func6.2({0x7ffbb42ca660, 0xc000975b00})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:118\n                | nodePoolParams.Replicas = int32(2)\n                | \n                \u003e err = tc.CreateNodePoolFromParam(ctx,\n                | \t*resourceGroup.Name,\n                | \tcustomerClusterName,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213c0?, 0xc000975b00?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 93\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:124 @ 03/01/26 03:41:53.656\n  \"ts\"=\"2026-03-01 03:41:53.656795\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-01 03:41:53.656865\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-autoscaling-hcp-cluster-pfg4sv\" \"resourceGroup\"=\"autoscaling-cluster-6mbww8\"\n  \"ts\"=\"2026-03-01 03:41:53.882690\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-autoscaling-hcp-cluster-pfg4sv\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-01 03:41:55.474450\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-31a71a35884d87a4a1de36eceeb324011e1c775154472da33ffc40cde64c08cb.yaml.gz\"\n  \"ts\"=\"2026-03-01 03:41:55.474569\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-31a71a35884d87a4a1de36eceeb324011e1c775154472da33ffc40cde64c08cb.yaml.gz\"\n  \"ts\"=\"2026-03-01 03:41:55.474638\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-01 03:41:55.475586\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-148\"\n  \"ts\"=\"2026-03-01 03:41:55.475602\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"autoscaling-cluster-6mbww8\"\n  \"ts\"=\"2026-03-01 03:41:55.655617\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"autoscaling-cluster-6mbww8\" \"deployment\"=\"pooledNonMsiScopedAssignments-e4ixwyxsfjnje\"\n  \"ts\"=\"2026-03-01 03:41:55.814591\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"autoscaling-cluster-6mbww8\" \"deployment\"=\"customer-infra-autoscaling-hcp-cluster-pfg4sv\"\n  \"ts\"=\"2026-03-01 03:41:55.974500\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-148\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-01 03:41:56.147010\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-148\" \"deployment\"=\"msi-deployment-148\"\n  \"ts\"=\"2026-03-01 03:41:56.300640\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-01 03:41:56.300813\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-01 03:41:56.300956\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"autoscaling-cluster-6mbww8\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","odepool with autoscaling enabled (Spec Runtime: 1h29m59.958s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36\n        In [It] (Node Runtime: 1h29m59.958s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36\n          At [By Step] creating the autoscaling nodepool (Step Runtime: 19m22.214s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:95\n\n          Spec Goroutine\n          goroutine 103 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x820bb58, 0xc0008fc370}, 0x77f1f7b?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x824f7a0, {0x820bb58, 0xc0008fc370}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateNodePoolAndWait({0x820bbc8, 0xc000578930}, 0xc000095788, {0xc000f0ee10, 0x15}, {0x78183e3, 0x14}, {0x77f1f57, 0xc}, {0xc001523630, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:757\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateNodePoolFromParam(0xc0007b37a0, {0x7f78b44bda60, 0xc000a0b200}, {0xc000f0ee10, 0x15}, {0x78183e3, 0x14}, {{0x77d5d51, 0x6}, {0x78183e3, ...}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:384\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func23.2({0x7f78b44bda60, 0xc000a0b200})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:96\n                | \n                | By(\"creating the autoscaling nodepool\")\n                \u003e err = tc.CreateNodePoolFromParam(ctx,\n                | \t*resourceGroup.Name,\n                | \tcustomerClusterName,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213c0?, 0xc000a0b200?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 100\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102 @ 03/01/26 03:41:54.92\n  \"ts\"=\"2026-03-01 03:41:54.920840\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-01 03:41:54.920909\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-autoscale-cluster-q7mds5\" \"resourceGroup\"=\"np-autoscaling-ntl9rp\"\n  \"ts\"=\"2026-03-01 03:41:55.097361\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-autoscale-cluster-q7mds5\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-01 03:41:56.239629\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n  \"ts\"=\"2026-03-01 03:41:56.239729\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n  \"ts\"=\"2026-03-01 03:41:56.239818\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-01 03:41:56.240822\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-116\"\n  \"ts\"=\"2026-03-01 03:41:56.240836\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"np-autoscaling-ntl9rp\"\n  \"ts\"=\"2026-03-01 03:41:56.385178\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-116\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-01 03:41:56.408437\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-ntl9rp\" \"deployment\"=\"pooledNonMsiScopedAssignments-ssrwqrnhyyabs\"\n  \"ts\"=\"2026-03-01 03:41:56.534892\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-116\" \"deployment\"=\"msi-deployment-116\"\n  \"ts\"=\"2026-03-01 03:41:56.745301\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-ntl9rp\" \"deployment\"=\"customer-infra-np-autoscale-cluster-q7mds5\"\n  \"ts\"=\"2026-03-01 03:41:56.911097\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-01 03:41:56.911200\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-01 03:41:56.911310\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"np-autoscaling-ntl9rp\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nodepool with autoscaling enabled (Spec Runtime: 1h29m59.958s)\n        /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36\n        In [It] (Node Runtime: 1h29m59.958s)\n          /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36\n          At [By Step] creating the autoscaling nodepool (Step Runtime: 19m22.214s)\n            /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:95\n\n          Spec Goroutine\n          goroutine 103 [select]\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x820bb58, 0xc0008fc370}, 0x77f1f7b?)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n            github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x824f7a0, {0x820bb58, 0xc0008fc370}, 0x0)\n              /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n            github.com/Azure/ARO-HCP/test/util/framework.CreateNodePoolAndWait({0x820bbc8, 0xc000578930}, 0xc000095788, {0xc000f0ee10, 0x15}, {0x78183e3, 0x14}, {0x77f1f57, 0xc}, {0xc001523630, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:757\n            github.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateNodePoolFromParam(0xc0007b37a0, {0x7f78b44bda60, 0xc000a0b200}, {0xc000f0ee10, 0x15}, {0x78183e3, 0x14}, {{0x77d5d51, 0x6}, {0x78183e3, ...}, ...}, ...)\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:384\n          \u003e github.com/Azure/ARO-HCP/test/e2e.init.func23.2({0x7f78b44bda60, 0xc000a0b200})\n              /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:96\n                | \n                | By(\"creating the autoscaling nodepool\")\n                \u003e err = tc.CreateNodePoolFromParam(ctx,\n                | \t*resourceGroup.Name,\n                | \tcustomerClusterName,\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x82213c0?, 0xc000a0b200?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 100\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102 @ 03/01/26 03:41:54.92\n  \"ts\"=\"2026-03-01 03:41:54.920840\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-01 03:41:54.920909\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-autoscale-cluster-q7mds5\" \"resourceGroup\"=\"np-autoscaling-ntl9rp\"\n  \"ts\"=\"2026-03-01 03:41:55.097361\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-autoscale-cluster-q7mds5\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-01 03:41:56.239629\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n  \"ts\"=\"2026-03-01 03:41:56.239729\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n  \"ts\"=\"2026-03-01 03:41:56.239818\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-01 03:41:56.240822\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-116\"\n  \"ts\"=\"2026-03-01 03:41:56.240836\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"np-autoscaling-ntl9rp\"\n  \"ts\"=\"2026-03-01 03:41:56.385178\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-116\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-01 03:41:56.408437\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-ntl9rp\" \"deployment\"=\"pooledNonMsiScopedAssignments-ssrwqrnhyyabs\"\n  \"ts\"=\"2026-03-01 03:41:56.534892\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-116\" \"deployment\"=\"msi-deployment-116\"\n  \"ts\"=\"2026-03-01 03:41:56.745301\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-ntl9rp\" \"deployment\"=\"customer-infra-np-autoscale-cluster-q7mds5\"\n  \"ts\"=\"2026-03-01 03:41:56.911097\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-01 03:41:56.911200\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-01 03:41:56.911310\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"np-autoscaling-ntl9rp\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027927018498166784","occurred_at":"2026-03-01T02:00:52Z","signature_id":"113cf5bd9bd7a7323070e6d41ca3636e8a47f1acd04d702ddfc76243f4e7d6f8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027927018498166784","occurred_at":"2026-03-01T02:00:52Z","signature_id":"c309f3c6a72783de7ec10a90294bfb08be7a4bb2a0d45720df0a65b378749a57","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f1386f529db6e94f57caba5baeea25a7ed546b620757afe7e7438f6ec564ee32","failure_pattern":"CreateOrUpdateExternalAuthAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for external auth \"ea-list\" in resourcegroup=\"ea-list-6jmnbl\" for cluster=\"ea-listr2rlb7\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-listr2rlb7 in resource group ea-list-6jmnbl, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[0,0,0,1,1,1,1],"trend_range":"2026-02-23..2026-03-01 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:118]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001106de0\u003e: \n    failed waiting for external auth \"ea-list\" in resourcegroup=\"ea-list-6jmnbl\" for cluster=\"ea-listr2rlb7\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-listr2rlb7 in resource group ea-list-6jmnbl, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"ea-list\\\" in resourcegroup=\\\"ea-list-6jmnbl\\\" for cluster=\\\"ea-listr2rlb7\\\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-listr2rlb7 in resource group ea-list-6jmnbl, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0008d0390\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-listr2rlb7 in resource group ea-list-6jmnbl\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:118]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001106de0\u003e: \n    failed waiting for external auth \"ea-list\" in resourcegroup=\"ea-list-6jmnbl\" for cluster=\"ea-listr2rlb7\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-listr2rlb7 in resource group ea-list-6jmnbl, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"ea-list\\\" in resourcegroup=\\\"ea-list-6jmnbl\\\" for cluster=\\\"ea-listr2rlb7\\\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-listr2rlb7 in resource group ea-list-6jmnbl, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0008d0390\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-listr2rlb7 in resource group ea-list-6jmnbl\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027927018498166784","occurred_at":"2026-03-01T02:00:52Z","signature_id":"94f92dfbf46d0a9f4f9a4a129d5064742d00a595a6be8b5cda3391a62c7b736d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"79d555cc6e5bd17f4b96af0c9e2da6a571d8241aacf20f84304c98773ef2682b","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-w8wfrk\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-02-23..2026-03-01 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:190]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0008101a0\u003e: \n    failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-w8wfrk\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed waiting for nodepool=\\\"np-ver-4-19-7\\\" for cluster \\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-w8wfrk\\\" to finish creating: context deadline exceeded\",\n        err: \u003ccontext.deadlineExceededError\u003e{},\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:190]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0008101a0\u003e: \n    failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-w8wfrk\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed waiting for nodepool=\\\"np-ver-4-19-7\\\" for cluster \\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-w8wfrk\\\" to finish creating: context deadline exceeded\",\n        err: \u003ccontext.deadlineExceededError\u003e{},\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2027927018498166784","occurred_at":"2026-03-01T02:00:52Z","signature_id":"a9aee60eba15b6c92fa33463f235ece4967dab3308e1f54d21f28b982f20aa35","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}