{"meta":{"date":"2026-07-03","timezone":"UTC","generated_at":"2026-08-28T03:08:39Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":4,"failed_runs":3,"runs_with_occurrences":3,"runs_with_failure_pattern_matches":3,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":1,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2073180037212803072","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-03T23:00:13Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2072984234015854592","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-03T10:02:08Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"37fdc910c1570135424aec2968eb689310dfc2114b99cb5b1b002d27b12de53e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2072984234015854592","occurred_at":"2026-07-03T10:02:08Z","failed_at":"e2e","signature_id":"e4791aafc091b76eba20acf25ac9fe7211505338fc604ecc4a37f045238c72a5","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-lwpsmq/cluster-candidate-4-22-dqkclz should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015228a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-dqkclz, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-dqkclz in resource group rg-candidate-4-22-lwpsmq, error: failed waiting for cluster=\"cluster-candidate-4-22-dqkclz\" in resourcegroup=\"rg-candidate-4-22-lwpsmq\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-dqkclz in resource group rg-candidate-4-22-lwpsmq, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-lwpsmq/cluster-candidate-4-22-dqkclz should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015228a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-dqkclz, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-dqkclz in resource group rg-candidate-4-22-lwpsmq, error: failed waiting for cluster=\"cluster-candidate-4-22-dqkclz\" in resourcegroup=\"rg-candidate-4-22-lwpsmq\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-dqkclz in resource group rg-candidate-4-22-lwpsmq, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-22-bmbbmp, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-bmbbmp in resource group rg-candidate-4-22-mdt4sp, error: failed waiting for cluster=\"cluster-candidate-4-22-bmbbmp\" in resourcegroup=\"rg-candidate-4-22-mdt4sp\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-bmbbmp in resource group rg-candidate-4-22-mdt4sp, error: context deadline exceeded"}},{"row_id":"3a61d0e5f6359220adf3a74e9e3af654d45e9dae0bd8d7d7aaff3b3f19ac75e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2072984234015854592","occurred_at":"2026-07-03T10:02:08Z","failed_at":"e2e","signature_id":"eb53a4a9c26a434d9e906d917ff2a534e915deff890ab198f961edde79f16c32","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-n5tq6q\" for cluster \"cluster-candidate-4-21-n5tq6q\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0028e9920\u003e: \n    failed to create NodePool np-n5tq6q, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-n5tq6q in resource group rg-candidate-4-21-hwrssb, error: failed waiting for nodepool=\"np-n5tq6q\" for cluster \"cluster-candidate-4-21-n5tq6q\" in resourcegroup=\"rg-candidate-4-21-hwrssb\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-n5tq6q\" for cluster \"cluster-candidate-4-21-n5tq6q\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0028e9920\u003e: \n    failed to create NodePool np-n5tq6q, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-n5tq6q in resource group rg-candidate-4-21-hwrssb, error: failed waiting for nodepool=\"np-n5tq6q\" for cluster \"cluster-candidate-4-21-n5tq6q\" in resourcegroup=\"rg-candidate-4-21-hwrssb\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-5nb65f, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5nb65f\" to finish creating: context deadline exceeded"}},{"row_id":"5230467c8c33d9867e3abf7258b448dc25ffbb383b2abd8fe0a83f7558927013","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2072984234015854592","occurred_at":"2026-07-03T10:02:08Z","failed_at":"e2e","signature_id":"e87b5e656d369390be6955142c66abfb62cb6be6a4065789e0b53a37cd133d33","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-p87pwt/cluster-candidate-5-0-xss2vb should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016e2e10\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-xss2vb, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-xss2vb in resource group rg-candidate-5-0-p87pwt, error: failed waiting for cluster=\"cluster-candidate-5-0-xss2vb\" in resourcegroup=\"rg-candidate-5-0-p87pwt\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-xss2vb in resource group rg-candidate-5-0-p87pwt, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-p87pwt/cluster-candidate-5-0-xss2vb should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0016e2e10\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-xss2vb, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-xss2vb in resource group rg-candidate-5-0-p87pwt, error: failed waiting for cluster=\"cluster-candidate-5-0-xss2vb\" in resourcegroup=\"rg-candidate-5-0-p87pwt\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-xss2vb in resource group rg-candidate-5-0-p87pwt, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-22-bmbbmp, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-bmbbmp in resource group rg-candidate-4-22-mdt4sp, error: failed waiting for cluster=\"cluster-candidate-4-22-bmbbmp\" in resourcegroup=\"rg-candidate-4-22-mdt4sp\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-bmbbmp in resource group rg-candidate-4-22-mdt4sp, error: context deadline exceeded"}},{"row_id":"64d296be7802b722f95629216c57de7e9b8168747d97e30bed40c7ff7545ad72","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2072984234015854592","occurred_at":"2026-07-03T10:02:08Z","failed_at":"e2e","signature_id":"ff39b4a4c6b06d8085da65733d7e2ae36f95e4f642503f5695535adfbac3f447","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-zwrxjs\" for cluster \"cluster-candidate-4-20-zwrxjs\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001d1aea0\u003e: \n    failed to create NodePool np-zwrxjs, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-zwrxjs in resource group rg-candidate-4-20-ctdjcb, error: failed waiting for nodepool=\"np-zwrxjs\" for cluster \"cluster-candidate-4-20-zwrxjs\" in resourcegroup=\"rg-candidate-4-20-ctdjcb\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-zwrxjs\" for cluster \"cluster-candidate-4-20-zwrxjs\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001d1aea0\u003e: \n    failed to create NodePool np-zwrxjs, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-zwrxjs in resource group rg-candidate-4-20-ctdjcb, error: failed waiting for nodepool=\"np-zwrxjs\" for cluster \"cluster-candidate-4-20-zwrxjs\" in resourcegroup=\"rg-candidate-4-20-ctdjcb\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-5nb65f, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5nb65f\" to finish creating: context deadline exceeded"}},{"row_id":"9ecbd9705a7ed3b9e30cba23e34d96c554e46fdec29562ba19646eec59cdd3c7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2072984234015854592","occurred_at":"2026-07-03T10:02:08Z","failed_at":"e2e","signature_id":"7b3755ad7640fb6701b0b7de05914f30ff5419bc2d62b28e3f707518980e9b19","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:148]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002990930\u003e: \n    failed to create NodePool autoscale-az, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-68bxll, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-68bxll\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:148]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002990930\u003e: \n    failed to create NodePool autoscale-az, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-68bxll, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-68bxll\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-5nb65f, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5nb65f\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072862922618441728","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-03T02:00:07Z"},"failed_at":["e2e"],"failed_test_count":14,"occurrences":[{"row_id":"074cfa6da431a233dddbf1ac8b52179405925a1596d1a937a72e76d8d803a9b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072862922618441728","occurred_at":"2026-07-03T02:00:07Z","failed_at":"e2e","signature_id":"e088db9f57479d836f086e3c41832593d4ca7353039976fb52e7925a1e61542f","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001046138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-fd7wxc\" in resourcegroup=\"rg-np-version-upgrade-fd7wxc-h6kd44\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001046138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-fd7wxc\" in resourcegroup=\"rg-np-version-upgrade-fd7wxc-h6kd44\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f0e44476112ce4e675f687843407c03482d80d265239de5b2ddbd45c441a04e","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-j8lscw\" to finish deleting: context canceled"}},{"row_id":"12d67f8f80472836bc9158733c893577833d2f42098be37ebe129f56ba4e4358","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072862922618441728","occurred_at":"2026-07-03T02:00:07Z","failed_at":"e2e","signature_id":"7fdfb9e1d1e996f1ac6eb2cce095dada7e3bccc0f32d0798f2c7b7747851bcff","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000c124c8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-h4hb29\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000c124c8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-h4hb29\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f0e44476112ce4e675f687843407c03482d80d265239de5b2ddbd45c441a04e","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-j8lscw\" to finish deleting: context canceled"}},{"row_id":"160785bd14005f8e413e5081e1d7fd871e3d5c1b2b39eb5ed43a72507fef0844","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072862922618441728","occurred_at":"2026-07-03T02:00:07Z","failed_at":"e2e","signature_id":"2d7234d13263e7adbdc8c24995f8d458870326c76fa69c8656d2e41a4cd6b954","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-vg4hqk\" for cluster \"cluster-candidate-4-20-vg4hqk\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0024a34d0\u003e: \n    failed to create NodePool np-vg4hqk, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-vg4hqk in resource group rg-candidate-4-20-hq9wqg, error: failed waiting for nodepool=\"np-vg4hqk\" for cluster \"cluster-candidate-4-20-vg4hqk\" in resourcegroup=\"rg-candidate-4-20-hq9wqg\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-vg4hqk\" for cluster \"cluster-candidate-4-20-vg4hqk\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0024a34d0\u003e: \n    failed to create NodePool np-vg4hqk, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-vg4hqk in resource group rg-candidate-4-20-hq9wqg, error: failed waiting for nodepool=\"np-vg4hqk\" for cluster \"cluster-candidate-4-20-vg4hqk\" in resourcegroup=\"rg-candidate-4-20-hq9wqg\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-5nb65f, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5nb65f\" to finish creating: context deadline exceeded"}},{"row_id":"17fc9367a0a9ef45387247217c57933435b43b04f1b141dc9fe61cf5779ef68e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072862922618441728","occurred_at":"2026-07-03T02:00:07Z","failed_at":"e2e","signature_id":"160a58c201446df1576b49ea8769c84da9360021ec90cf0cbd781907c02a75ac","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001430228\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-j8lscw\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001430228\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-j8lscw\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f0e44476112ce4e675f687843407c03482d80d265239de5b2ddbd45c441a04e","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-j8lscw\" to finish deleting: context canceled"}},{"row_id":"228c67a4c5f148dbc3c9b8d8a473a2330a596d085d1151e8e87857da9942acfa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072862922618441728","occurred_at":"2026-07-03T02:00:07Z","failed_at":"e2e","signature_id":"0fbeef3547c7d201c9f582b65c22f0f63780b8eb3e174efec59112bfdf0770ff","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001338360\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-fxvbg6\" in resourcegroup=\"rg-np-skip-minor-fxvbg6-pggjsw\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001338360\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-fxvbg6\" in resourcegroup=\"rg-np-skip-minor-fxvbg6-pggjsw\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f0e44476112ce4e675f687843407c03482d80d265239de5b2ddbd45c441a04e","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-j8lscw\" to finish deleting: context canceled"}},{"row_id":"252d1df6e970b2ad99f46d46f47f34e6a00093b4696689c1419280d254a42774","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072862922618441728","occurred_at":"2026-07-03T02:00:07Z","failed_at":"e2e","signature_id":"b07b35943f37f8d1158d2b693424287fdb312b6db24972bd300d4291e3acf626","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:136]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001be1dd0\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-fncw96, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-fncw96\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:136]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001be1dd0\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-fncw96, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-fncw96\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-5nb65f, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5nb65f\" to finish creating: context deadline exceeded"}},{"row_id":"43b18e4a016b71a5ba69b7862cee3e47dbb241976f432f6e374169d7bcbda854","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072862922618441728","occurred_at":"2026-07-03T02:00:07Z","failed_at":"e2e","signature_id":"6903394e640d5d2f2b33ff361336b5aab8602a4436c8607b025cc750d5c8d169","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013ee480\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-4qk94s, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4qk94s\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0013ee480\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-4qk94s, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4qk94s\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-5nb65f, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5nb65f\" to finish creating: context deadline exceeded"}},{"row_id":"6a2bf5a850a7455558f48bc734476d785297d21d17895423d2330a3b1de4d380","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072862922618441728","occurred_at":"2026-07-03T02:00:07Z","failed_at":"e2e","signature_id":"6d2a4aee54e31529569c81e7b4a12ec11b0f07cd20fffed7c77a18f28b909522","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00273e2e8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-ckrxk9\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc00273e2e8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-ckrxk9\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f0e44476112ce4e675f687843407c03482d80d265239de5b2ddbd45c441a04e","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-j8lscw\" to finish deleting: context canceled"}},{"row_id":"8ab402e79cb969c0b33b108c523b1c87ae4b565a334512707c25502596392535","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072862922618441728","occurred_at":"2026-07-03T02:00:07Z","failed_at":"e2e","signature_id":"6775652474047a81e5c5589d1ee106b19998f9b6cca67114890173ead8ff5a6d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-cl6857/cluster-candidate-4-22-nvlv6d should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010b00c0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-nvlv6d, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-nvlv6d in resource group rg-candidate-4-22-cl6857, error: failed waiting for cluster=\"cluster-candidate-4-22-nvlv6d\" in resourcegroup=\"rg-candidate-4-22-cl6857\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-nvlv6d in resource group rg-candidate-4-22-cl6857, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-cl6857/cluster-candidate-4-22-nvlv6d should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010b00c0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-nvlv6d, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-nvlv6d in resource group rg-candidate-4-22-cl6857, error: failed waiting for cluster=\"cluster-candidate-4-22-nvlv6d\" in resourcegroup=\"rg-candidate-4-22-cl6857\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-nvlv6d in resource group rg-candidate-4-22-cl6857, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-22-bmbbmp, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-bmbbmp in resource group rg-candidate-4-22-mdt4sp, error: failed waiting for cluster=\"cluster-candidate-4-22-bmbbmp\" in resourcegroup=\"rg-candidate-4-22-mdt4sp\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-bmbbmp in resource group rg-candidate-4-22-mdt4sp, error: context deadline exceeded"}},{"row_id":"a23c888e301e2ec321f046026dc37992c3d67cd47f633c3f7f6e4228957a4898","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072862922618441728","occurred_at":"2026-07-03T02:00:07Z","failed_at":"e2e","signature_id":"959f11b9cff2639227e47ea2d7833f461ed367da62b7f723de11db15781820d0","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:148]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002a03c50\u003e: \n    failed to create NodePool autoscale-az, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-5nb65f, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5nb65f\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:148]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002a03c50\u003e: \n    failed to create NodePool autoscale-az, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-5nb65f, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5nb65f\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-5nb65f, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5nb65f\" to finish creating: context deadline exceeded"}},{"row_id":"c054afab6ac200ede6b3ca76ab52e558c7f26818f1ddc678a9f85513de9953b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072862922618441728","occurred_at":"2026-07-03T02:00:07Z","failed_at":"e2e","signature_id":"319a3f8f5ed60de7ca0587f49554bae3f01b3992c4e11b15f31c0b51facf340c","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.21.zLatest to 4.19.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000ece3d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-dg-minor-gtd6pd\" in resourcegroup=\"rg-np-dg-minor-gtd6pd-wtnrz7\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc000ece3d8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-dg-minor-gtd6pd\" in resourcegroup=\"rg-np-dg-minor-gtd6pd-wtnrz7\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f0e44476112ce4e675f687843407c03482d80d265239de5b2ddbd45c441a04e","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-j8lscw\" to finish deleting: context canceled"}},{"row_id":"c795e163e0e0b4b91ac7fb7b9db1cfe58f5b6f496ed92a2ff02b4bd12e7e9249","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072862922618441728","occurred_at":"2026-07-03T02:00:07Z","failed_at":"e2e","signature_id":"cd7a59d906a512f866ab9802ee73c7b8a8355c05fc4da03a23b42de611fdb2de","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0010d0708\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-jszpnx\" in resourcegroup=\"rg-np-version-upgrade-jszpnx-sbzxq2\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0010d0708\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-jszpnx\" in resourcegroup=\"rg-np-version-upgrade-jszpnx-sbzxq2\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f0e44476112ce4e675f687843407c03482d80d265239de5b2ddbd45c441a04e","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-j8lscw\" to finish deleting: context canceled"}},{"row_id":"cc223ba136360f8333b27328d12d4bcc977c2222b80bcac3bfd2062f1bfadd7a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072862922618441728","occurred_at":"2026-07-03T02:00:07Z","failed_at":"e2e","signature_id":"2b8b29513bbe30abfd5144bb8628183bea3f0d7741ea8cfd5555e31c25ae01b3","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0012ec258\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-downgrade-gncrr7\" in resourcegroup=\"rg-np-downgrade-gncrr7-fmwxc7\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc0012ec258\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-downgrade-gncrr7\" in resourcegroup=\"rg-np-downgrade-gncrr7-fmwxc7\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f0e44476112ce4e675f687843407c03482d80d265239de5b2ddbd45c441a04e","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-j8lscw\" to finish deleting: context canceled"}},{"row_id":"e171867274383864b60434bb51e319a5e947939e873a3972ef46c37d0eb494c5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2072862922618441728","occurred_at":"2026-07-03T02:00:07Z","failed_at":"e2e","signature_id":"ed2779cff79b25b7c6f641b6762101368aeff4579496e95ceaca2949f14ebf09","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001252048\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-noedge-mz9frr\" in resourcegroup=\"rg-np-noedge-mz9frr-9l95c5\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283] A node timeout occurred and then the following failure was recorded in the timedout node before it exited:\nUnexpected error:\n    \u003c*errors.joinError | 0xc001252048\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-noedge-mz9frr\" in resourcegroup=\"rg-np-noedge-mz9frr-9l95c5\" to finish deleting: context canceled\n    ...\noccurred  \nfail [:0]: A node timeout occurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7f0e44476112ce4e675f687843407c03482d80d265239de5b2ddbd45c441a04e","failure_pattern":"failed to cleanup resource group: at least one HCP cluster \u003ccluster\u003e to delete: failed waiting for hcpcluster=\"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish deleting: context canceled","search_query":"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-j8lscw\" to finish deleting: context canceled"}}],"failure_pattern_summary":{"signature_count":14,"failure_pattern_ids":["6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","7f0e44476112ce4e675f687843407c03482d80d265239de5b2ddbd45c441a04e","941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0"],"matched_occurrences":14,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2072837998688342016","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-03T00:21:04Z"},"failed_at":["e2e"],"failed_test_count":6,"occurrences":[{"row_id":"0c3126a1567d42cfefe677253af65fcc4b3f939f6d0681b7dcc66959a0f31c44","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2072837998688342016","occurred_at":"2026-07-03T00:21:04Z","failed_at":"e2e","signature_id":"499d2386809dfcb37497aba91766c4846629fefc303dff202872af4507cbdeab","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:136]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00232b710\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-lgvgbh, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-lgvgbh\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:136]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00232b710\u003e: \n    failed to create NodePool np-1, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-lgvgbh, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-lgvgbh\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-5nb65f, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5nb65f\" to finish creating: context deadline exceeded"}},{"row_id":"330c95683bd0299629cd812c5f7858fa755897c8f1bc967a46fc8570bcea2e51","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2072837998688342016","occurred_at":"2026-07-03T00:21:04Z","failed_at":"e2e","signature_id":"26c85f0ff77847219c6829e08fb3c4d8f6dd9f4978093c539bfe78dae7d357ca","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-qb294n\" for cluster \"cluster-candidate-4-21-qb294n\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0027ec870\u003e: \n    failed to create NodePool np-qb294n, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-qb294n in resource group rg-candidate-4-21-9csn4d, error: failed waiting for nodepool=\"np-qb294n\" for cluster \"cluster-candidate-4-21-qb294n\" in resourcegroup=\"rg-candidate-4-21-9csn4d\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-qb294n\" for cluster \"cluster-candidate-4-21-qb294n\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0027ec870\u003e: \n    failed to create NodePool np-qb294n, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-qb294n in resource group rg-candidate-4-21-9csn4d, error: failed waiting for nodepool=\"np-qb294n\" for cluster \"cluster-candidate-4-21-qb294n\" in resourcegroup=\"rg-candidate-4-21-9csn4d\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-5nb65f, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5nb65f\" to finish creating: context deadline exceeded"}},{"row_id":"56ca4baedb87ea7fbefe1367b3364a46b501c2a7640b0f3ae9bb307481f51bb9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2072837998688342016","occurred_at":"2026-07-03T00:21:04Z","failed_at":"e2e","signature_id":"01b3a7fb1e9c47f78ab4be711401d88c7e525753aba44bbd2b5407127de5bf57","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-mdt4sp/cluster-candidate-4-22-bmbbmp should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015e68d0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-bmbbmp, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-bmbbmp in resource group rg-candidate-4-22-mdt4sp, error: failed waiting for cluster=\"cluster-candidate-4-22-bmbbmp\" in resourcegroup=\"rg-candidate-4-22-mdt4sp\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-bmbbmp in resource group rg-candidate-4-22-mdt4sp, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-22-mdt4sp/cluster-candidate-4-22-bmbbmp should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015e68d0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-bmbbmp, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-bmbbmp in resource group rg-candidate-4-22-mdt4sp, error: failed waiting for cluster=\"cluster-candidate-4-22-bmbbmp\" in resourcegroup=\"rg-candidate-4-22-mdt4sp\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-bmbbmp in resource group rg-candidate-4-22-mdt4sp, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-22-bmbbmp, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-bmbbmp in resource group rg-candidate-4-22-mdt4sp, error: failed waiting for cluster=\"cluster-candidate-4-22-bmbbmp\" in resourcegroup=\"rg-candidate-4-22-mdt4sp\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-bmbbmp in resource group rg-candidate-4-22-mdt4sp, error: context deadline exceeded"}},{"row_id":"5f9cdbc93c2185143452f7c078f075cb8c7ef78bdc0edf05165c2b9975319a75","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2072837998688342016","occurred_at":"2026-07-03T00:21:04Z","failed_at":"e2e","signature_id":"648079fb156a0e0acfc524ea2a9ed34add9fdabf3afa6f4ee93fe2ebc6d9b927","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-2n2bn6\" for cluster \"cluster-candidate-4-20-2n2bn6\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0020ea5a0\u003e: \n    failed to create NodePool np-2n2bn6, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-2n2bn6 in resource group rg-candidate-4-20-sxqkm6, error: failed waiting for nodepool=\"np-2n2bn6\" for cluster \"cluster-candidate-4-20-2n2bn6\" in resourcegroup=\"rg-candidate-4-20-sxqkm6\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-2n2bn6\" for cluster \"cluster-candidate-4-20-2n2bn6\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0020ea5a0\u003e: \n    failed to create NodePool np-2n2bn6, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-2n2bn6 in resource group rg-candidate-4-20-sxqkm6, error: failed waiting for nodepool=\"np-2n2bn6\" for cluster \"cluster-candidate-4-20-2n2bn6\" in resourcegroup=\"rg-candidate-4-20-sxqkm6\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-5nb65f, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5nb65f\" to finish creating: context deadline exceeded"}},{"row_id":"7d736877d648c6d76342eeab6e6e6765ab80e86be9538be0cceaa727cae943f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2072837998688342016","occurred_at":"2026-07-03T00:21:04Z","failed_at":"e2e","signature_id":"3d06043ceb55b93529f0349cc22ecd27bb8b5c0111962579db0ced2e86cfb2f2","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-d5zr7j/cluster-candidate-5-0-rxmj56 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015f27b0\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-rxmj56, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-rxmj56 in resource group rg-candidate-5-0-d5zr7j, error: failed waiting for cluster=\"cluster-candidate-5-0-rxmj56\" in resourcegroup=\"rg-candidate-5-0-d5zr7j\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-rxmj56 in resource group rg-candidate-5-0-d5zr7j, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-5-0-d5zr7j/cluster-candidate-5-0-rxmj56 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015f27b0\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-rxmj56, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-rxmj56 in resource group rg-candidate-5-0-d5zr7j, error: failed waiting for cluster=\"cluster-candidate-5-0-rxmj56\" in resourcegroup=\"rg-candidate-5-0-d5zr7j\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-5-0-rxmj56 in resource group rg-candidate-5-0-d5zr7j, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-22-bmbbmp, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-bmbbmp in resource group rg-candidate-4-22-mdt4sp, error: failed waiting for cluster=\"cluster-candidate-4-22-bmbbmp\" in resourcegroup=\"rg-candidate-4-22-mdt4sp\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-22-bmbbmp in resource group rg-candidate-4-22-mdt4sp, error: context deadline exceeded"}},{"row_id":"bd60d3b616d2e823c07c102b825bb3f890f5e0985eb37f994ba13deacad5cbff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2072837998688342016","occurred_at":"2026-07-03T00:21:04Z","failed_at":"e2e","signature_id":"c7769378f6930988011a2c4e90884327880e12975aafd631f7a226a9f8346932","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:148]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0029151a0\u003e: \n    failed to create NodePool autoscale-az, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-gt6qbd, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-gt6qbd\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:148]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0029151a0\u003e: \n    failed to create NodePool autoscale-az, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-gt6qbd, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-gt6qbd\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool autoscale-az in resource group np-autoscaling-5nb65f, error: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5nb65f\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}