{"meta":{"start_date":"2026-03-05","end_date":"2026-03-05","start_at":"2026-03-05T00:00:00Z","end_at":"2026-03-06T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T22:24:23Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":5,"failed_runs":4,"raw_occurrences":20,"matched_occurrences":20,"runs_affected":4},"rows":[{"environment":"prod","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-mbd9lr, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-mbd9lr\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":7,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":7,"after_last_push_count":7,"trend_counts":[0,0,0,0,0,0,7],"trend_range":"2026-02-27..2026-03-05 UTC","prior_windows_present":1,"prior_window_starts":["2026-02-23"],"prior_runs_affected":8,"prior_last_seen_at":"2026-03-01T02:00:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014cc990\u003e: \n    failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-mbd9lr, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-mbd9lr\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-mbd9lr, error: failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-mbd9lr\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001298900\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-mbd9lr\",\n            },\n            \u003c*fmt.wrapError | 0xc000dce820\u003e{\n                msg: \"failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-mbd9lr\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014cc990\u003e: \n    failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-mbd9lr, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-mbd9lr\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-mbd9lr, error: failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-mbd9lr\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc001298900\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-mbd9lr\",\n            },\n            \u003c*fmt.wrapError | 0xc000dce820\u003e{\n                msg: \"failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-mbd9lr\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:118]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ef68d0\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group basic-cluster-sgkhjw, error: failed waiting for nodepool=\"np-1\" for cluster \"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-sgkhjw\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group basic-cluster-sgkhjw, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-sgkhjw\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0004deee0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group basic-cluster-sgkhjw\",\n            },\n            \u003c*fmt.wrapError | 0xc000517520\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-sgkhjw\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create.go:118]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ef68d0\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group basic-cluster-sgkhjw, error: failed waiting for nodepool=\"np-1\" for cluster \"basic-hcp-cluster\" in resourcegroup=\"basic-cluster-sgkhjw\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group basic-cluster-sgkhjw, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-sgkhjw\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0004deee0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group basic-cluster-sgkhjw\",\n            },\n            \u003c*fmt.wrapError | 0xc000517520\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"basic-hcp-cluster\\\" in resourcegroup=\\\"basic-cluster-sgkhjw\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00137f290\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-hlwxtl, error: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-hlwxtl\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-hlwxtl, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-hlwxtl\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00068dd20\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-hlwxtl\",\n            },\n            \u003c*fmt.wrapError | 0xc000d26be0\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-hlwxtl\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:112]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc00137f290\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-hlwxtl, error: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-hlwxtl\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-hlwxtl, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-hlwxtl\\\" to finish creating: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00068dd20\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-hlwxtl\",\n            },\n            \u003c*fmt.wrapError | 0xc000d26be0\u003e{\n                msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-hlwxtl\\\" to finish creating: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2029376488976945152","occurred_at":"2026-03-05T02:00:33Z","signature_id":"5690b098eeee6cfa4a1eccae3f8aec88d077e51f4a17cb2857a86cf6a7b8964e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2029376488976945152","occurred_at":"2026-03-05T02:00:33Z","signature_id":"57eccb44a481863c56e957a0bd8ca9b5e89364a25b54e86056e9d7ed9fd1715d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2029376488976945152","occurred_at":"2026-03-05T02:00:33Z","signature_id":"6657dd318ad8f3428b2a83bf882b4057b608597ce58b085d43e63aef4ffd5da6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2029376488976945152","occurred_at":"2026-03-05T02:00:33Z","signature_id":"9a0a20937dc28e1207c41666144d7fcff1b045c2bb7a2145640eda0ef69311d1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2029376488976945152","occurred_at":"2026-03-05T02:00:33Z","signature_id":"a9d01c9f62f6996df7625ae4872b82ede3e47014c995511c4a98108dfd51cd48","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2029376488976945152","occurred_at":"2026-03-05T02:00:33Z","signature_id":"c6865c823f4fbd95ac29eba0a60199eba4afdf297670023556af87082d68624c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2029376488976945152","occurred_at":"2026-03-05T02:00:33Z","signature_id":"d88396eb2fc95cafa9c8d065888dfbbe91cfda3e268ae88959786a20b58a0f13","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-mzqcvh, error: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-mzqcvh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster basic-cluster in resource group rg-cluster-nsg-subnet-reuse-mzqcvh, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":3,"failed_runs":3,"run_impact_percent":60,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,0,5],"trend_range":"2026-02-27..2026-03-05 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to list HCP clusters without node pools at both subscription and resource group levels","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e4e540\u003e: \n    failed to create HCP cluster list-test-cluster-jfhd8m, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-jfhd8m in resource group cluster-listing-dtc5mn, error: failed waiting for cluster=\"list-test-cluster-jfhd8m\" in resourcegroup=\"cluster-listing-dtc5mn\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-jfhd8m in resource group cluster-listing-dtc5mn, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster list-test-cluster-jfhd8m, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-jfhd8m in resource group cluster-listing-dtc5mn, error: failed waiting for cluster=\\\"list-test-cluster-jfhd8m\\\" in resourcegroup=\\\"cluster-listing-dtc5mn\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-jfhd8m in resource group cluster-listing-dtc5mn, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000ea4ad0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-jfhd8m in resource group cluster-listing-dtc5mn\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000e4e510\u003e{\n                msg: \"failed waiting for cluster=\\\"list-test-cluster-jfhd8m\\\" in resourcegroup=\\\"cluster-listing-dtc5mn\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-jfhd8m in resource group cluster-listing-dtc5mn, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000ea4ad0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-jfhd8m in resource group cluster-listing-dtc5mn\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:86]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e4e540\u003e: \n    failed to create HCP cluster list-test-cluster-jfhd8m, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-jfhd8m in resource group cluster-listing-dtc5mn, error: failed waiting for cluster=\"list-test-cluster-jfhd8m\" in resourcegroup=\"cluster-listing-dtc5mn\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-jfhd8m in resource group cluster-listing-dtc5mn, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster list-test-cluster-jfhd8m, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-jfhd8m in resource group cluster-listing-dtc5mn, error: failed waiting for cluster=\\\"list-test-cluster-jfhd8m\\\" in resourcegroup=\\\"cluster-listing-dtc5mn\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-jfhd8m in resource group cluster-listing-dtc5mn, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000ea4ad0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-jfhd8m in resource group cluster-listing-dtc5mn\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000e4e510\u003e{\n                msg: \"failed waiting for cluster=\\\"list-test-cluster-jfhd8m\\\" in resourcegroup=\\\"cluster-listing-dtc5mn\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-jfhd8m in resource group cluster-listing-dtc5mn, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000ea4ad0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster list-test-cluster-jfhd8m in resource group cluster-listing-dtc5mn\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010d6bd0\u003e: \n    failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-q9vcm9, error: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-q9vcm9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-q9vcm9, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-q9vcm9, error: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-q9vcm9\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-q9vcm9, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00107ebc0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-q9vcm9\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0010d6ba0\u003e{\n                msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-q9vcm9\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-q9vcm9, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc00107ebc0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-q9vcm9\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010d6bd0\u003e: \n    failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-q9vcm9, error: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-q9vcm9\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-q9vcm9, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-q9vcm9, error: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-q9vcm9\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-q9vcm9, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc00107ebc0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-q9vcm9\",\n            },\n            \u003c*fmt.wrapErrors | 0xc0010d6ba0\u003e{\n                msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-q9vcm9\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-q9vcm9, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc00107ebc0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-q9vcm9\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:100]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000fba750\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-p52sxb, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-p52sxb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-p52sxb, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-p52sxb, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-p52sxb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-p52sxb, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000f253e0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-p52sxb\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000fba720\u003e{\n                msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-p52sxb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-p52sxb, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000f253e0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-p52sxb\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:100]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000fba750\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-p52sxb, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-p52sxb\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-p52sxb, error: context deadline exceeded\n    {\n        msg: \"failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-p52sxb, error: failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-p52sxb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-p52sxb, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000f253e0\u003e{\n                s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-p52sxb\",\n            },\n            \u003c*fmt.wrapErrors | 0xc000fba720\u003e{\n                msg: \"failed waiting for cluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-p52sxb\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-p52sxb, error: context deadline exceeded\",\n                errs: [\n                    \u003c*errors.errorString | 0xc000f253e0\u003e{\n                        s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-p52sxb\",\n                    },\n                    \u003ccontext.deadlineExceededError\u003e{},\n                ],\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2029525294263570432","occurred_at":"2026-03-05T11:51:51Z","signature_id":"52ffc9148ddaae130657f47d5ee56a315ef660020c8361603a1b60c5edad38e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2029525294263570432","occurred_at":"2026-03-05T11:51:51Z","signature_id":"f829270bdf903a2b97369dac4358487abf05f9460f9622a96a53fa0ac95640b8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2029458659259453440","occurred_at":"2026-03-05T07:27:04Z","signature_id":"80d52a655f083a028c97396d295f36d5f609e9385e91c676e5d3ba7bf0528fbf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2029458659259453440","occurred_at":"2026-03-05T07:27:04Z","signature_id":"cf9cde3b58af8c9a6ea2ea97066e0cbab1e3a2e76509fa82ff634ea29998ac5f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2029429994882600960","occurred_at":"2026-03-05T05:33:10Z","signature_id":"cb42fe43476eab329e7e7a2c24419a1076aab3e37fadfaa69db7200545a24082","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":40,"anchor_occurrences":2,"after_last_push_count":16,"trend_counts":[2,3,2,1,5,1,2],"trend_range":"2026-02-27..2026-03-05 UTC","prior_windows_present":1,"prior_window_starts":["2026-02-23"],"prior_runs_affected":10,"prior_last_seen_at":"2026-03-01T02:00:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["o complete creation\")\n                \u003e _, err = poller2.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n                | \tFrequency: framework.StandardPollInterval,\n                | })\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8221420?, 0xc000d270b0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 48\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:135 @ 03/05/26 07:05:43.617\n  \"ts\"=\"2026-03-05 07:05:43.617714\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-05 07:05:43.617781\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-kwnth4\" \"resourceGroup\"=\"customer-rg-hxctbd\"\n  \"ts\"=\"2026-03-05 07:05:43.829234\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-kwnth4\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-05 07:05:45.615032\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-4clpsq\" \"resourceGroup\"=\"customer-rg-hxctbd\"\n  \"ts\"=\"2026-03-05 07:05:45.785098\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-4clpsq\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-05 07:05:47.508736\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-05 07:05:47.508844\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-05 07:05:47.508918\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-05 07:05:47.509851\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-108\"\n  \"ts\"=\"2026-03-05 07:05:47.509863\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-124\"\n  \"ts\"=\"2026-03-05 07:05:47.509936\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-hxctbd\"\n  \"ts\"=\"2026-03-05 07:05:47.509891\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-120\"\n  \"ts\"=\"2026-03-05 07:05:47.705727\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-hxctbd\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n  \"ts\"=\"2026-03-05 07:05:47.837833\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-hxctbd\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-4clpsq\"\n  \"ts\"=\"2026-03-05 07:05:47.842262\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-120\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-05 07:05:47.843623\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-124\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-05 07:05:47.971538\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-120\" \"deployment\"=\"msi-deployment-120\"\n  \"ts\"=\"2026-03-05 07:05:47.971609\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-124\" \"deployment\"=\"msi-deployment-124\"\n  \"ts\"=\"2026-03-05 07:05:47.979881\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-108\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-05 07:05:47.993788\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-hxctbd\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-03-05 07:05:48.130080\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-hxctbd\" \"deployment\"=\"customer-infra-basic-hcp-cluster-kwnth4\"\n  \"ts\"=\"2026-03-05 07:05:48.635410\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-108\" \"deployment\"=\"msi-deployment-108\"\n  \"ts\"=\"2026-03-05 07:05:48.772313\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-05 07:05:48.772502\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-05 07:05:48.772619\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-hxctbd\"\n  \"ts\"=\"2026-03-05 07:20:34.392780\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"customer-rg-hxctbd\"\n  \"ts\"=\"2026-03-05 07:20:34.392812\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"customer-rg-hxctbd\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\no complete creation\")\n                \u003e _, err = poller2.PollUntilDone(ctx, \u0026runtime.PollUntilDoneOptions{\n                | \tFrequency: framework.StandardPollInterval,\n                | })\n            github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8221420?, 0xc000d270b0?})\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n            github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 48\n              /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n    ------------------------------\n    [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:135 @ 03/05/26 07:05:43.617\n  \"ts\"=\"2026-03-05 07:05:43.617714\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-03-05 07:05:43.617781\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster-kwnth4\" \"resourceGroup\"=\"customer-rg-hxctbd\"\n  \"ts\"=\"2026-03-05 07:05:43.829234\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster-kwnth4\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-05 07:05:45.615032\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-4clpsq\" \"resourceGroup\"=\"customer-rg-hxctbd\"\n  \"ts\"=\"2026-03-05 07:05:45.785098\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-basic-hcp-cluster2-4clpsq\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-03-05 07:05:47.508736\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-05 07:05:47.508844\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-b8ed2189eb62b4145bd17832fa47015d3fc1ad56caa60a743b61b98540ac41f3.yaml.gz\"\n  \"ts\"=\"2026-03-05 07:05:47.508918\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-03-05 07:05:47.509851\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-108\"\n  \"ts\"=\"2026-03-05 07:05:47.509863\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-124\"\n  \"ts\"=\"2026-03-05 07:05:47.509936\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"customer-rg-hxctbd\"\n  \"ts\"=\"2026-03-05 07:05:47.509891\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-120\"\n  \"ts\"=\"2026-03-05 07:05:47.705727\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-hxctbd\" \"deployment\"=\"pooledNonMsiScopedAssignments-mooiin3edme32\"\n  \"ts\"=\"2026-03-05 07:05:47.837833\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-hxctbd\" \"deployment\"=\"customer-infra-basic-hcp-cluster2-4clpsq\"\n  \"ts\"=\"2026-03-05 07:05:47.842262\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-120\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-05 07:05:47.843623\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-124\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-05 07:05:47.971538\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-120\" \"deployment\"=\"msi-deployment-120\"\n  \"ts\"=\"2026-03-05 07:05:47.971609\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-124\" \"deployment\"=\"msi-deployment-124\"\n  \"ts\"=\"2026-03-05 07:05:47.979881\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-108\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-03-05 07:05:47.993788\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-hxctbd\" \"deployment\"=\"pooledNonMsiScopedAssignments-eqbgltuyzhpg4\"\n  \"ts\"=\"2026-03-05 07:05:48.130080\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"customer-rg-hxctbd\" \"deployment\"=\"customer-infra-basic-hcp-cluster-kwnth4\"\n  \"ts\"=\"2026-03-05 07:05:48.635410\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-108\" \"deployment\"=\"msi-deployment-108\"\n  \"ts\"=\"2026-03-05 07:05:48.772313\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-03-05 07:05:48.772502\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-03-05 07:05:48.772619\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"customer-rg-hxctbd\"\n  \"ts\"=\"2026-03-05 07:20:34.392780\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"customer-rg-hxctbd\"\n  \"ts\"=\"2026-03-05 07:20:34.392812\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"customer-rg-hxctbd\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command",".(*clientConnReadLoop).run(0xc000383fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:2258 +0xca fp=0xc000383f60 sp=0xc000383e18 pc=0x31a62aa\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc001550380)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:2127 +0x52 fp=0xc000383fc8 sp=0xc000383f60 pc=0x31a5632\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:880 +0x25 fp=0xc000383fe0 sp=0xc000383fc8 pc=0x319edc5\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000383fe8 sp=0xc000383fe0 pc=0x2ac5921\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 95\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:880 +0xda5\n\ngoroutine 1155 gp=0xc001550a80 m=nil [IO wait]:\nruntime.gopark(0xc0008a3600?, 0xc000e11758?, 0x6c?, 0xa0?, 0xb?)\n\t/usr/local/go/src/runtime/proc.go:460 +0xce fp=0xc000e11720 sp=0xc000e11700 pc=0x2abd28e\nruntime.netpollblock(0x2b26d58?, 0x2a49046?, 0x0?)\n\t/usr/local/go/src/runtime/netpoll.go:575 +0xf7 fp=0xc000e11758 sp=0xc000e11720 pc=0x2a7ce97\ninternal/poll.runtime_pollWait(0x7f281ec6ca00, 0x72)\n\t/usr/local/go/src/runtime/netpoll.go:351 +0x85 fp=0xc000e11778 sp=0xc000e11758 pc=0x2abbb25\ninternal/poll.(*pollDesc).wait(0xc000251d80?, 0xc00148c000?, 0x0)\n\t/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc000e117a0 sp=0xc000e11778 pc=0x2b49a87\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/local/go/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc000251d80, {0xc00148c000, 0x1300, 0x1300})\n\t/usr/local/go/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc000e11838 sp=0xc000e117a0 pc=0x2b4aeb9\nnet.(*netFD).Read(0xc000251d80, {0xc00148c000?, 0xc00148c000?, 0x5?})\n\t/usr/local/go/src/net/fd_posix.go:68 +0x25 fp=0xc000e11880 sp=0xc000e11838 pc=0x2cdf185\nnet.(*conn).Read(0xc0009360c8, {0xc00148c000?, 0x7f286585c210?, 0x7f28658535c0?})\n\t/usr/local/go/src/net/net.go:196 +0x45 fp=0xc000e118c8 sp=0xc000e11880 pc=0x2cf1225\ncrypto/tls.(*atLeastReader).Read(0xc000013b60, {0xc00148c000?, 0x12fb?, 0xe?})\n\t/usr/local/go/src/crypto/tls/conn.go:819 +0x3b fp=0xc000e11910 sp=0xc000e118c8 pc=0x2d5cadb\nbytes.(*Buffer).ReadFrom(0xc00095d428, {0x8173120, 0xc000013b60})\n\t/usr/local/go/src/bytes/buffer.go:217 +0x98 fp=0xc000e11968 sp=0xc000e11910 pc=0x2b87a38\ncrypto/tls.(*Conn).readFromUntil(0xc00095d188, {0x8173b20, 0xc0009360c8}, 0xc000e119d0?)\n\t/usr/local/go/src/crypto/tls/conn.go:841 +0xde fp=0xc000e119a0 sp=0xc000e11968 pc=0x2d5ccbe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc00095d188, 0x0)\n\t/usr/local/go/src/crypto/tls/conn.go:630 +0x3db fp=0xc000e11c18 sp=0xc000e119a0 pc=0x2d59d3b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/local/go/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc00095d188, {0xc001478000, 0x1000, 0x319b220?})\n\t/usr/local/go/src/crypto/tls/conn.go:1397 +0x145 fp=0xc000e11c80 sp=0xc000e11c18 pc=0x2d60625\nbufio.(*Reader).Read(0xc000e81440, {0xc00154e2e4, 0x9, 0x2a409ab?})\n\t/usr/local/go/src/bufio/bufio.go:245 +0x197 fp=0xc000e11cb8 sp=0xc000e11c80 pc=0x2c22617\nio.ReadAtLeast({0x8172700, 0xc000e81440}, {0xc00154e2e4, 0x9, 0x9}, 0x9)\n\t/usr/local/go/src/io/io.go:335 +0x8e fp=0xc000e11d00 sp=0xc000e11cb8 pc=0x2b17aee\nio.ReadFull(...)\n\t/usr/local/go/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc00154e2e4, 0x9, 0x6cd24e0?}, {0x8172700?, 0xc000e81440?})\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/frame.go:242 +0x65 fp=0xc000e11d50 sp=0xc000e11d00 pc=0x31929c5\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc00154e2a0)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/frame.go:505 +0x6b fp=0xc000e11dd8 sp=0xc000e11d50 pc=0x319358b\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc00154e2a0)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/frame.go:564 +0x18 fp=0xc000e11e18 sp=0xc000e11dd8 pc=0x3193cf8\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000e11fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:2258 +0xca fp=0xc000e11f60 sp=0xc000e11e18 pc=0x31a62aa\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc001551c00)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:2127 +0x52 fp=0xc000e11fc8 sp=0xc000e11f60 pc=0x31a5632\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:880 +0x25 fp=0xc000e11fe0 sp=0xc000e11fc8 pc=0x319edc5\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000e11fe8 sp=0xc000e11fe0 pc=0x2ac5921\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 1154\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:880 +0xda5\n\nrax    0xfffffffffffffffc\nrbx    0x5\nrcx    0x2a3ce0e\nrdx    0x80\nrdi    0x5\nrsi    0x7ffff925c09c\nrbp    0x7ffff925c050\nrsp    0x7ffff925c008\nr8     0x0\nr9     0x0\nr10    0x11ba\nr11    0x246\nr12    0x7ffff925c0e0\nr13    0x1\nr14    0xb2ed360\nr15    0x5\nrip    0x2a3ce0e\nrflags 0x246\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\n.(*clientConnReadLoop).run(0xc000383fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:2258 +0xca fp=0xc000383f60 sp=0xc000383e18 pc=0x31a62aa\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc001550380)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:2127 +0x52 fp=0xc000383fc8 sp=0xc000383f60 pc=0x31a5632\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:880 +0x25 fp=0xc000383fe0 sp=0xc000383fc8 pc=0x319edc5\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000383fe8 sp=0xc000383fe0 pc=0x2ac5921\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 95\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:880 +0xda5\n\ngoroutine 1155 gp=0xc001550a80 m=nil [IO wait]:\nruntime.gopark(0xc0008a3600?, 0xc000e11758?, 0x6c?, 0xa0?, 0xb?)\n\t/usr/local/go/src/runtime/proc.go:460 +0xce fp=0xc000e11720 sp=0xc000e11700 pc=0x2abd28e\nruntime.netpollblock(0x2b26d58?, 0x2a49046?, 0x0?)\n\t/usr/local/go/src/runtime/netpoll.go:575 +0xf7 fp=0xc000e11758 sp=0xc000e11720 pc=0x2a7ce97\ninternal/poll.runtime_pollWait(0x7f281ec6ca00, 0x72)\n\t/usr/local/go/src/runtime/netpoll.go:351 +0x85 fp=0xc000e11778 sp=0xc000e11758 pc=0x2abbb25\ninternal/poll.(*pollDesc).wait(0xc000251d80?, 0xc00148c000?, 0x0)\n\t/usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc000e117a0 sp=0xc000e11778 pc=0x2b49a87\ninternal/poll.(*pollDesc).waitRead(...)\n\t/usr/local/go/src/internal/poll/fd_poll_runtime.go:89\ninternal/poll.(*FD).Read(0xc000251d80, {0xc00148c000, 0x1300, 0x1300})\n\t/usr/local/go/src/internal/poll/fd_unix.go:165 +0x279 fp=0xc000e11838 sp=0xc000e117a0 pc=0x2b4aeb9\nnet.(*netFD).Read(0xc000251d80, {0xc00148c000?, 0xc00148c000?, 0x5?})\n\t/usr/local/go/src/net/fd_posix.go:68 +0x25 fp=0xc000e11880 sp=0xc000e11838 pc=0x2cdf185\nnet.(*conn).Read(0xc0009360c8, {0xc00148c000?, 0x7f286585c210?, 0x7f28658535c0?})\n\t/usr/local/go/src/net/net.go:196 +0x45 fp=0xc000e118c8 sp=0xc000e11880 pc=0x2cf1225\ncrypto/tls.(*atLeastReader).Read(0xc000013b60, {0xc00148c000?, 0x12fb?, 0xe?})\n\t/usr/local/go/src/crypto/tls/conn.go:819 +0x3b fp=0xc000e11910 sp=0xc000e118c8 pc=0x2d5cadb\nbytes.(*Buffer).ReadFrom(0xc00095d428, {0x8173120, 0xc000013b60})\n\t/usr/local/go/src/bytes/buffer.go:217 +0x98 fp=0xc000e11968 sp=0xc000e11910 pc=0x2b87a38\ncrypto/tls.(*Conn).readFromUntil(0xc00095d188, {0x8173b20, 0xc0009360c8}, 0xc000e119d0?)\n\t/usr/local/go/src/crypto/tls/conn.go:841 +0xde fp=0xc000e119a0 sp=0xc000e11968 pc=0x2d5ccbe\ncrypto/tls.(*Conn).readRecordOrCCS(0xc00095d188, 0x0)\n\t/usr/local/go/src/crypto/tls/conn.go:630 +0x3db fp=0xc000e11c18 sp=0xc000e119a0 pc=0x2d59d3b\ncrypto/tls.(*Conn).readRecord(...)\n\t/usr/local/go/src/crypto/tls/conn.go:592\ncrypto/tls.(*Conn).Read(0xc00095d188, {0xc001478000, 0x1000, 0x319b220?})\n\t/usr/local/go/src/crypto/tls/conn.go:1397 +0x145 fp=0xc000e11c80 sp=0xc000e11c18 pc=0x2d60625\nbufio.(*Reader).Read(0xc000e81440, {0xc00154e2e4, 0x9, 0x2a409ab?})\n\t/usr/local/go/src/bufio/bufio.go:245 +0x197 fp=0xc000e11cb8 sp=0xc000e11c80 pc=0x2c22617\nio.ReadAtLeast({0x8172700, 0xc000e81440}, {0xc00154e2e4, 0x9, 0x9}, 0x9)\n\t/usr/local/go/src/io/io.go:335 +0x8e fp=0xc000e11d00 sp=0xc000e11cb8 pc=0x2b17aee\nio.ReadFull(...)\n\t/usr/local/go/src/io/io.go:354\ngolang.org/x/net/http2.readFrameHeader({0xc00154e2e4, 0x9, 0x6cd24e0?}, {0x8172700?, 0xc000e81440?})\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/frame.go:242 +0x65 fp=0xc000e11d50 sp=0xc000e11d00 pc=0x31929c5\ngolang.org/x/net/http2.(*Framer).ReadFrameHeader(0xc00154e2a0)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/frame.go:505 +0x6b fp=0xc000e11dd8 sp=0xc000e11d50 pc=0x319358b\ngolang.org/x/net/http2.(*Framer).ReadFrame(0xc00154e2a0)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/frame.go:564 +0x18 fp=0xc000e11e18 sp=0xc000e11dd8 pc=0x3193cf8\ngolang.org/x/net/http2.(*clientConnReadLoop).run(0xc000e11fa8)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:2258 +0xca fp=0xc000e11f60 sp=0xc000e11e18 pc=0x31a62aa\ngolang.org/x/net/http2.(*ClientConn).readLoop(0xc001551c00)\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:2127 +0x52 fp=0xc000e11fc8 sp=0xc000e11f60 pc=0x31a5632\ngolang.org/x/net/http2.(*Transport).newClientConn.gowrap1()\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:880 +0x25 fp=0xc000e11fe0 sp=0xc000e11fc8 pc=0x319edc5\nruntime.goexit({})\n\t/usr/local/go/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000e11fe8 sp=0xc000e11fe0 pc=0x2ac5921\ncreated by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 1154\n\t/go/pkg/mod/golang.org/x/net@v0.49.0/http2/transport.go:880 +0xda5\n\nrax    0xfffffffffffffffc\nrbx    0x5\nrcx    0x2a3ce0e\nrdx    0x80\nrdi    0x5\nrsi    0x7ffff925c09c\nrbp    0x7ffff925c050\nrsp    0x7ffff925c008\nr8     0x0\nr9     0x0\nr10    0x11ba\nr11    0x246\nr12    0x7ffff925c0e0\nr13    0x1\nr14    0xb2ed360\nr15    0x5\nrip    0x2a3ce0e\nrflags 0x246\ncs     0x33\nfs     0x0\ngs     0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2029429994882600960","occurred_at":"2026-03-05T05:33:10Z","signature_id":"16aae9966619744a64c2127b89cb918e309289be14807542bf7ca858e6a4adaf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2029376488976945152","occurred_at":"2026-03-05T02:00:33Z","signature_id":"11375158fa80661caa725c8868bcdfeb28ce475cb78fc7e105120f8d3ff5a283","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":40,"anchor_occurrences":2,"after_last_push_count":33,"trend_counts":[1,0,0,1,12,16,2],"trend_range":"2026-02-27..2026-03-05 UTC","prior_windows_present":1,"prior_window_starts":["2026-02-23"],"prior_runs_affected":9,"prior_last_seen_at":"2026-02-27T02:00:51Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2029429994882600960","occurred_at":"2026-03-05T05:33:10Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2029376488976945152","occurred_at":"2026-03-05T02:00:33Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,2,0,1],"trend_range":"2026-02-27..2026-03-05 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.004s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.004s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: false\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2029458659259453440","occurred_at":"2026-03-05T07:27:04Z","signature_id":"ee8099a3b414c84f0dabbe6ee3f745157c59a94f539cde4bae37fe8fe9711731","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f1386f529db6e94f57caba5baeea25a7ed546b620757afe7e7438f6ec564ee32","failure_pattern":"CreateOrUpdateExternalAuthAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for external auth \"ea-list\" in resourcegroup=\"ea-list-dzsdxm\" for cluster=\"ea-listxfk7fg\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-listxfk7fg in resource group ea-list-dzsdxm, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-02-27..2026-03-05 UTC","prior_windows_present":1,"prior_window_starts":["2026-02-23"],"prior_runs_affected":8,"prior_last_seen_at":"2026-03-01T02:00:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:118]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000877800\u003e: \n    failed waiting for external auth \"ea-list\" in resourcegroup=\"ea-list-dzsdxm\" for cluster=\"ea-listxfk7fg\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-listxfk7fg in resource group ea-list-dzsdxm, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"ea-list\\\" in resourcegroup=\\\"ea-list-dzsdxm\\\" for cluster=\\\"ea-listxfk7fg\\\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-listxfk7fg in resource group ea-list-dzsdxm, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0008746a0\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-listxfk7fg in resource group ea-list-dzsdxm\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:118]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000877800\u003e: \n    failed waiting for external auth \"ea-list\" in resourcegroup=\"ea-list-dzsdxm\" for cluster=\"ea-listxfk7fg\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-listxfk7fg in resource group ea-list-dzsdxm, error: context deadline exceeded\n    {\n        msg: \"failed waiting for external auth \\\"ea-list\\\" in resourcegroup=\\\"ea-list-dzsdxm\\\" for cluster=\\\"ea-listxfk7fg\\\" to finish, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-listxfk7fg in resource group ea-list-dzsdxm, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0008746a0\u003e{\n                s: \"timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ea-list in cluster ea-listxfk7fg in resource group ea-list-dzsdxm\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2029376488976945152","occurred_at":"2026-03-05T02:00:33Z","signature_id":"6d4a0053e53017bc2efb5fec419a9339144dbc625f26148d16e0f6c9ec31ae88","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a4836f65199e80fe4c6e326686d84c1284118526f09ab7a96fa6064477b81554","failure_pattern":"ERROR CODE: Conflict; detail message Cannot request credential while resource is \"provisioning\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-02-27..2026-03-05 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:103]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00049ee40\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-4v2vfj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot request credential while resource is \\\"provisioning\\\"\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-4v2vfj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-4v2vfj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Cannot request credential while resource is \\\\\\\"provisioning\\\\\\\"\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-4v2vfj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc00147a210\u003e{\n            ErrorCode: \"Conflict\",\n            StatusCode: 409,\n            RawResponse: {\n                Status: \"409 Conflict\",\n                StatusCode: 409,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Envoy-Upstream-Service-Time\": [\"3041\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/5f33cbb9-f37a-429b-bce1-8edaf9908424\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260305T021539Z:547cbd7f-379b-40c1-bc2c-84c59906fb55\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: AD38B254178B410585C028F8AB33DE1C Ref B: CH1AA2020610031 Ref C: 2026-03-05T02:15:35Z\",\n                    ],\n                    \"Date\": [\n                        \"Thu, 05 Mar 2026 02:15:38 GMT\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"X-Ms-Error-Code\": [\"Conflict\"],\n                    \"X-Ms-Request-Id\": [\n                        \"b2ef27e1-1259-4b08-a852-40ef41f5feca\",\n                    ],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"547cbd7f-379b-40c1-bc2c-84c59906fb55\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Content-Length\": [\"319\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"Expires\": [\"-1\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc00049ece0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 114, 101, 113, 117, 101, 115, 116, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 32, 119, 104, 105, 108, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 92, 34, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 92, 34, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 52, 48, 51, 100, 57, 100, 101, 57, 45, 49, 51, 50, 98, 45, 52, 57, 55, 52, 45, 57, 52, 97, 53, 45, 53, 98, 55, 56, 98, 100, 102, 97, 49, 57, 49, 101...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:103]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00049ee40\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-4v2vfj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 409: 409 Conflict\n    ERROR CODE: Conflict\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"Conflict\",\n        \"message\": \"Cannot request credential while resource is \\\"provisioning\\\"\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-4v2vfj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-4v2vfj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"Conflict\\\",\\n    \\\"message\\\": \\\"Cannot request credential while resource is \\\\\\\"provisioning\\\\\\\"\\\",\\n    \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-4v2vfj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc00147a210\u003e{\n            ErrorCode: \"Conflict\",\n            StatusCode: 409,\n            RawResponse: {\n                Status: \"409 Conflict\",\n                StatusCode: 409,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Envoy-Upstream-Service-Time\": [\"3041\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/5f33cbb9-f37a-429b-bce1-8edaf9908424\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"UKSOUTH:20260305T021539Z:547cbd7f-379b-40c1-bc2c-84c59906fb55\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: AD38B254178B410585C028F8AB33DE1C Ref B: CH1AA2020610031 Ref C: 2026-03-05T02:15:35Z\",\n                    ],\n                    \"Date\": [\n                        \"Thu, 05 Mar 2026 02:15:38 GMT\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"X-Ms-Error-Code\": [\"Conflict\"],\n                    \"X-Ms-Request-Id\": [\n                        \"b2ef27e1-1259-4b08-a852-40ef41f5feca\",\n                    ],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"547cbd7f-379b-40c1-bc2c-84c59906fb55\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Content-Length\": [\"319\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"Expires\": [\"-1\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc00049ece0\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 67, 111, 110, 102, 108, 105, 99, 116, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 67, 97, 110, 110, 111, 116, 32, 114, 101, 113, 117, 101, 115, 116, 32, 99, 114, 101, 100, 101, 110, 116, 105, 97, 108, 32, 119, 104, 105, 108, 101, 32, 114, 101, 115, 111, 117, 114, 99, 101, 32, 105, 115, 32, 92, 34, 112, 114, 111, 118, 105, 115, 105, 111, 110, 105, 110, 103, 92, 34, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 116, 97, 114, 103, 101, 116, 34, 58, 32, 34, 47, 115, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 110, 115, 47, 52, 48, 51, 100, 57, 100, 101, 57, 45, 49, 51, 50, 98, 45, 52, 57, 55, 52, 45, 57, 52, 97, 53, 45, 53, 98, 55, 56, 98, 100, 102, 97, 49, 57, 49, 101...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2029376488976945152","occurred_at":"2026-03-05T02:00:33Z","signature_id":"e0b0778584beff88c7177e5b89f0191dd4480c074d9c92e153c2195a5e38fc0c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"79d555cc6e5bd17f4b96af0c9e2da6a571d8241aacf20f84304c98773ef2682b","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-zqs4nb\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-02-27..2026-03-05 UTC","prior_windows_present":1,"prior_window_starts":["2026-02-23"],"prior_runs_affected":4,"prior_last_seen_at":"2026-03-01T02:00:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:190]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001195460\u003e: \n    failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-zqs4nb\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed waiting for nodepool=\\\"np-ver-4-19-7\\\" for cluster \\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-zqs4nb\\\" to finish creating: context deadline exceeded\",\n        err: \u003ccontext.deadlineExceededError\u003e{},\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:190]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc001195460\u003e: \n    failed waiting for nodepool=\"np-ver-4-19-7\" for cluster \"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-zqs4nb\" to finish creating: context deadline exceeded\n    {\n        msg: \"failed waiting for nodepool=\\\"np-ver-4-19-7\\\" for cluster \\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-zqs4nb\\\" to finish creating: context deadline exceeded\",\n        err: \u003ccontext.deadlineExceededError\u003e{},\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2029376488976945152","occurred_at":"2026-03-05T02:00:33Z","signature_id":"62bdb5cfa7eb3801845b141eb945bfc086977ab55b12f146d6260cef91cefdc6","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}