{"meta":{"start_date":"2026-06-13","end_date":"2026-06-13","start_at":"2026-06-13T00:00:00Z","end_at":"2026-06-14T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T09:14:45Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":2,"failed_runs":2,"raw_occurrences":16,"matched_occurrences":16,"runs_affected":2},"rows":[{"environment":"prod","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-mw5w9v, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-mw5w9v\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":8,"runs_affected":2,"failed_runs":2,"run_impact_percent":100,"anchor_occurrences":8,"after_last_push_count":8,"trend_counts":[0,0,0,0,0,0,8],"trend_range":"2026-06-07..2026-06-13 UTC","prior_windows_present":3,"prior_window_starts":["2026-05-11","2026-05-18","2026-05-25"],"prior_runs_affected":14,"prior_last_seen_at":"2026-05-27T23:14:22Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-6qggtl\" for cluster \"cluster-nightly-4-22-6qggtl\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0007f67e0\u003e: \n    failed to create NodePool np-6qggtl, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-6qggtl in resource group rg-nightly-4-22-rhdjmj, error: failed waiting for nodepool=\"np-6qggtl\" for cluster \"cluster-nightly-4-22-6qggtl\" in resourcegroup=\"rg-nightly-4-22-rhdjmj\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-6qggtl\" for cluster \"cluster-nightly-4-22-6qggtl\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0007f67e0\u003e: \n    failed to create NodePool np-6qggtl, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-6qggtl in resource group rg-nightly-4-22-rhdjmj, error: failed waiting for nodepool=\"np-6qggtl\" for cluster \"cluster-nightly-4-22-6qggtl\" in resourcegroup=\"rg-nightly-4-22-rhdjmj\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-f7q6bg\" for cluster \"cluster-nightly-4-21-f7q6bg\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000887050\u003e: \n    failed to create NodePool np-f7q6bg, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-f7q6bg in resource group rg-nightly-4-21-xxvpqx, error: failed waiting for nodepool=\"np-f7q6bg\" for cluster \"cluster-nightly-4-21-f7q6bg\" in resourcegroup=\"rg-nightly-4-21-xxvpqx\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-f7q6bg\" for cluster \"cluster-nightly-4-21-f7q6bg\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000887050\u003e: \n    failed to create NodePool np-f7q6bg, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-f7q6bg in resource group rg-nightly-4-21-xxvpqx, error: failed waiting for nodepool=\"np-f7q6bg\" for cluster \"cluster-nightly-4-21-f7q6bg\" in resourcegroup=\"rg-nightly-4-21-xxvpqx\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-mpxl94\" for cluster \"cluster-nightly-4-20-mpxl94\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0007d30b0\u003e: \n    failed to create NodePool np-mpxl94, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-mpxl94 in resource group rg-nightly-4-20-j4bp9d, error: failed waiting for nodepool=\"np-mpxl94\" for cluster \"cluster-nightly-4-20-mpxl94\" in resourcegroup=\"rg-nightly-4-20-j4bp9d\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-mpxl94\" for cluster \"cluster-nightly-4-20-mpxl94\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0007d30b0\u003e: \n    failed to create NodePool np-mpxl94, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-mpxl94 in resource group rg-nightly-4-20-j4bp9d, error: failed waiting for nodepool=\"np-mpxl94\" for cluster \"cluster-nightly-4-20-mpxl94\" in resourcegroup=\"rg-nightly-4-20-j4bp9d\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065932263165530112","occurred_at":"2026-06-13T23:00:09Z","signature_id":"224d5d874238f1cd22488873ca7190b72187df0d9fe09841e5db2b96275a1b4e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065932263165530112","occurred_at":"2026-06-13T23:00:09Z","signature_id":"88032920372bb580b92d64fe51e4bdd8991eda298f3795aa36cd7737c90b4a69","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065932263165530112","occurred_at":"2026-06-13T23:00:09Z","signature_id":"e2d6bd557c45f7b473546eb7111383b1daa7ca88a22b4701e0ee86e877890da0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065932263165530112","occurred_at":"2026-06-13T23:00:09Z","signature_id":"f5d8698bd794108aedcde836540090a583a69ebbaa1fe948c20bc81ecc35b80e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065615170868088832","occurred_at":"2026-06-13T02:00:08Z","signature_id":"2af954d36535a0a7481035736c9a91d83a0f0ffc777925839aa2f87413518070","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065615170868088832","occurred_at":"2026-06-13T02:00:08Z","signature_id":"3bc8e0373dbd9e1f19bfd4306bfcbe51c1682a324b18c0e03ed2b11acfa65b40","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065615170868088832","occurred_at":"2026-06-13T02:00:08Z","signature_id":"b85582a724900abf00f1a29eb4d59920e3a06c0530e620bc8245baeefeeeb491","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065615170868088832","occurred_at":"2026-06-13T02:00:08Z","signature_id":"f8053c826e6cd91fead74e70a0aa2d6016090d2ff2b76651bb137a4f3a22b9e5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-nightly-4-23-2pc4mf\" in resourcegroup=\"rg-nightly-4-23-l2rvdl\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-2pc4mf in resource group rg-nightly-4-23-l2rvdl, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":4,"runs_affected":2,"failed_runs":2,"run_impact_percent":100,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-06-07..2026-06-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-05-11","2026-05-18","2026-05-25","2026-06-01"],"prior_runs_affected":38,"prior_last_seen_at":"2026-06-07T23:00:01Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-l2rvdl/cluster-nightly-4-23-2pc4mf should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000c10b10\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-2pc4mf\" in resourcegroup=\"rg-nightly-4-23-l2rvdl\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-2pc4mf in resource group rg-nightly-4-23-l2rvdl, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-l2rvdl/cluster-nightly-4-23-2pc4mf should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000c10b10\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-2pc4mf\" in resourcegroup=\"rg-nightly-4-23-l2rvdl\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-2pc4mf in resource group rg-nightly-4-23-l2rvdl, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-8fnj8w/cluster-nightly-5-0-pc6pg4 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0005eec60\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-pc6pg4\" in resourcegroup=\"rg-nightly-5-0-8fnj8w\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-pc6pg4 in resource group rg-nightly-5-0-8fnj8w, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-8fnj8w/cluster-nightly-5-0-pc6pg4 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0005eec60\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-pc6pg4\" in resourcegroup=\"rg-nightly-5-0-8fnj8w\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-pc6pg4 in resource group rg-nightly-5-0-8fnj8w, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-vcn7cv/cluster-nightly-5-0-9w646r should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0003084e0\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-9w646r\" in resourcegroup=\"rg-nightly-5-0-vcn7cv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-9w646r in resource group rg-nightly-5-0-vcn7cv, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-vcn7cv/cluster-nightly-5-0-9w646r should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0003084e0\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-9w646r\" in resourcegroup=\"rg-nightly-5-0-vcn7cv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-9w646r in resource group rg-nightly-5-0-vcn7cv, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065932263165530112","occurred_at":"2026-06-13T23:00:09Z","signature_id":"0efd7a11cb057b0aec96ab5bfbd67f9bed5d8540f42648f4d7182063d9538390","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065932263165530112","occurred_at":"2026-06-13T23:00:09Z","signature_id":"9ee000acddd0fba7399e9c0f0083b44860c3f29bc291b509a910dd1343206a75","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065615170868088832","occurred_at":"2026-06-13T02:00:08Z","signature_id":"4edf6fee90ed5858139054c814d406c16d5131fbc5ca8b622581771deb7e2c42","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065615170868088832","occurred_at":"2026-06-13T02:00:08Z","signature_id":"e8111712700769ba5ea4edd004a453254ff50e4d7e2a573f0974c341b5572219","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"7426c1a835b553e399edab9b5f3d89e2c53ed02e293edf09d1e41a3543dc8dec","failure_pattern":"ERROR CODE: InvalidTemplateDeployment; provider Microsoft.Compute","search_query":"ERROR CODE: InvalidTemplateDeployment","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":100,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,1,2],"trend_range":"2026-06-07..2026-06-13 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:130]: failed to deploy test VM\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010931e0\u003e: \n    failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-fjsk59\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-fjsk59/providers/Microsoft.Resources/deployments/test-vm\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplateDeployment\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplateDeployment\",\n        \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.Compute/virtualMachines (2023-03-01)' reported preflight validation errors. Tracking id is 'af5eb3b1-d28d-40aa-b393-7c5df027bafd'. See inner errors for details.\",\n        \"details\": [\n          {\n            \"code\": \"QuotaExceeded\",\n            \"message\": \"Operation could not be completed as it results in exceeding approved standardDSv5Family Cores quota. Additional details - Deployment Model: Resource Manager, Location: uksouth, Current Limit: 0, Current Usage: 0, Additional Required: 2, (Minimum) New Limit Required: 2. Setup Alerts when Quota reaches threshold. Learn more at https://aka.ms/quotamonitoringalerting . Submit a request for Quota increase at https://aka.ms/ProdportalCRP/#blade/Microsoft_Azure_Capacity/UsageAndQuota.ReactView/Parameters/%7B%22subscriptionId%22:%22XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%22,%22command%22:%22openQuotaApprovalBlade%22,%22quotas%22:[%7B%22location%22:%22uksouth%22,%22providerId%22:%22Microsoft.Compute%22,%22resourceName%22:%22standardDSv5Family%22,%22quotaRequest%22:%7B%22properties%22:%7B%22limit%22:2,%22unit%22:%22Count%22,%22name%22:%7B%22value%22:%22standardDSv5Family%22%7D%7D%7D%7D]%7D by specifying parameters listed in the ‘Details’ section for deployment to succeed. Please read more about quota limits at https://docs.microsoft.com/en-us/azure/azure-supportability/per-vm-quota-requests\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:130]: failed to deploy test VM\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010931e0\u003e: \n    failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-fjsk59\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-fjsk59/providers/Microsoft.Resources/deployments/test-vm\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplateDeployment\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplateDeployment\",\n        \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.Compute/virtualMachines (2023-03-01)' reported preflight validation errors. Tracking id is 'af5eb3b1-d28d-40aa-b393-7c5df027bafd'. See inner errors for details.\",\n        \"details\": [\n          {\n            \"code\": \"QuotaExceeded\",\n            \"message\": \"Operation could not be completed as it results in exceeding approved standardDSv5Family Cores quota. Additional details - Deployment Model: Resource Manager, Location: uksouth, Current Limit: 0, Current Usage: 0, Additional Required: 2, (Minimum) New Limit Required: 2. Setup Alerts when Quota reaches threshold. Learn more at https://aka.ms/quotamonitoringalerting . Submit a request for Quota increase at https://aka.ms/ProdportalCRP/#blade/Microsoft_Azure_Capacity/UsageAndQuota.ReactView/Parameters/%7B%22subscriptionId%22:%22XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%22,%22command%22:%22openQuotaApprovalBlade%22,%22quotas%22:[%7B%22location%22:%22uksouth%22,%22providerId%22:%22Microsoft.Compute%22,%22resourceName%22:%22standardDSv5Family%22,%22quotaRequest%22:%7B%22properties%22:%7B%22limit%22:2,%22unit%22:%22Count%22,%22name%22:%7B%22value%22:%22standardDSv5Family%22%7D%7D%7D%7D]%7D by specifying parameters listed in the ‘Details’ section for deployment to succeed. Please read more about quota limits at https://docs.microsoft.com/en-us/azure/azure-supportability/per-vm-quota-requests\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:130]: failed to deploy test VM\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010b9180\u003e: \n    failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-w9mrjk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-w9mrjk/providers/Microsoft.Resources/deployments/test-vm\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplateDeployment\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplateDeployment\",\n        \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.Compute/virtualMachines (2023-03-01)' reported preflight validation errors. Tracking id is '3e358d95-8993-4562-a5f9-f29fa3953f34'. See inner errors for details.\",\n        \"details\": [\n          {\n            \"code\": \"QuotaExceeded\",\n            \"message\": \"Operation could not be completed as it results in exceeding approved standardDSv5Family Cores quota. Additional details - Deployment Model: Resource Manager, Location: uksouth, Current Limit: 0, Current Usage: 0, Additional Required: 2, (Minimum) New Limit Required: 2. Setup Alerts when Quota reaches threshold. Learn more at https://aka.ms/quotamonitoringalerting . Submit a request for Quota increase at https://aka.ms/ProdportalCRP/#blade/Microsoft_Azure_Capacity/UsageAndQuota.ReactView/Parameters/%7B%22subscriptionId%22:%22XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%22,%22command%22:%22openQuotaApprovalBlade%22,%22quotas%22:[%7B%22location%22:%22uksouth%22,%22providerId%22:%22Microsoft.Compute%22,%22resourceName%22:%22standardDSv5Family%22,%22quotaRequest%22:%7B%22properties%22:%7B%22limit%22:2,%22unit%22:%22Count%22,%22name%22:%7B%22value%22:%22standardDSv5Family%22%7D%7D%7D%7D]%7D by specifying parameters listed in the ‘Details’ section for deployment to succeed. Please read more about quota limits at https://docs.microsoft.com/en-us/azure/azure-supportability/per-vm-quota-requests\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:130]: failed to deploy test VM\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010b9180\u003e: \n    failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-w9mrjk\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-w9mrjk/providers/Microsoft.Resources/deployments/test-vm\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplateDeployment\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplateDeployment\",\n        \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.Compute/virtualMachines (2023-03-01)' reported preflight validation errors. Tracking id is '3e358d95-8993-4562-a5f9-f29fa3953f34'. See inner errors for details.\",\n        \"details\": [\n          {\n            \"code\": \"QuotaExceeded\",\n            \"message\": \"Operation could not be completed as it results in exceeding approved standardDSv5Family Cores quota. Additional details - Deployment Model: Resource Manager, Location: uksouth, Current Limit: 0, Current Usage: 0, Additional Required: 2, (Minimum) New Limit Required: 2. Setup Alerts when Quota reaches threshold. Learn more at https://aka.ms/quotamonitoringalerting . Submit a request for Quota increase at https://aka.ms/ProdportalCRP/#blade/Microsoft_Azure_Capacity/UsageAndQuota.ReactView/Parameters/%7B%22subscriptionId%22:%22XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%22,%22command%22:%22openQuotaApprovalBlade%22,%22quotas%22:[%7B%22location%22:%22uksouth%22,%22providerId%22:%22Microsoft.Compute%22,%22resourceName%22:%22standardDSv5Family%22,%22quotaRequest%22:%7B%22properties%22:%7B%22limit%22:2,%22unit%22:%22Count%22,%22name%22:%7B%22value%22:%22standardDSv5Family%22%7D%7D%7D%7D]%7D by specifying parameters listed in the ‘Details’ section for deployment to succeed. Please read more about quota limits at https://docs.microsoft.com/en-us/azure/azure-supportability/per-vm-quota-requests\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065932263165530112","occurred_at":"2026-06-13T23:00:09Z","signature_id":"61247fbe34626c91d73d49f2f7d5f4f639487e390dce5a64af6ad1784287a7a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065615170868088832","occurred_at":"2026-06-13T02:00:08Z","signature_id":"e86a40108734edad969cf766bf9f1f0b9ffc44eb43d51e3e502976796202076f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":100,"anchor_occurrences":2,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-06-07..2026-06-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-05-18"],"prior_runs_affected":4,"prior_last_seen_at":"2026-05-21T20:06:27Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000df03f0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-bn486z, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-bn486z\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000df03f0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-bn486z, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-bn486z\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00181a420\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-vdrfcr, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-vdrfcr\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00181a420\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-vdrfcr, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-vdrfcr\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2065932263165530112","occurred_at":"2026-06-13T23:00:09Z","signature_id":"f173a978707f057f0927ba957cfb5fa902ce5c16fe0653d9f33e5a374fdb405d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2065615170868088832","occurred_at":"2026-06-13T02:00:08Z","signature_id":"63c9271b9e105dce0242df2a05a76a9e669a791aac70d4e5117c2fd4b66e1497","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}