{"meta":{"start_date":"2026-06-15","end_date":"2026-06-15","start_at":"2026-06-15T00:00:00Z","end_at":"2026-06-16T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T09:15:56Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":4,"failed_runs":3,"raw_occurrences":18,"matched_occurrences":18,"runs_affected":3},"rows":[{"environment":"prod","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-2mfzr6, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-2mfzr6\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":9,"runs_affected":2,"failed_runs":2,"run_impact_percent":50,"anchor_occurrences":9,"after_last_push_count":9,"trend_counts":[0,0,0,0,0,0,9],"trend_range":"2026-06-09..2026-06-15 UTC","prior_windows_present":3,"prior_window_starts":["2026-05-18","2026-05-25","2026-06-08"],"prior_runs_affected":17,"prior_last_seen_at":"2026-06-14T23:00:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-8nxjrh\" for cluster \"cluster-nightly-4-20-8nxjrh\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a2d350\u003e: \n    failed to create NodePool np-8nxjrh, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-8nxjrh in resource group rg-nightly-4-20-gm65vp, error: failed waiting for nodepool=\"np-8nxjrh\" for cluster \"cluster-nightly-4-20-8nxjrh\" in resourcegroup=\"rg-nightly-4-20-gm65vp\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-8nxjrh\" for cluster \"cluster-nightly-4-20-8nxjrh\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a2d350\u003e: \n    failed to create NodePool np-8nxjrh, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-8nxjrh in resource group rg-nightly-4-20-gm65vp, error: failed waiting for nodepool=\"np-8nxjrh\" for cluster \"cluster-nightly-4-20-8nxjrh\" in resourcegroup=\"rg-nightly-4-20-gm65vp\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-86tbzv\" for cluster \"cluster-nightly-4-22-86tbzv\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011cba10\u003e: \n    failed to create NodePool np-86tbzv, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-86tbzv in resource group rg-nightly-4-22-xqf2d4, error: failed waiting for nodepool=\"np-86tbzv\" for cluster \"cluster-nightly-4-22-86tbzv\" in resourcegroup=\"rg-nightly-4-22-xqf2d4\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:192]: failed to create node pool \"np-86tbzv\" for cluster \"cluster-nightly-4-22-86tbzv\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011cba10\u003e: \n    failed to create NodePool np-86tbzv, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-86tbzv in resource group rg-nightly-4-22-xqf2d4, error: failed waiting for nodepool=\"np-86tbzv\" for cluster \"cluster-nightly-4-22-86tbzv\" in resourcegroup=\"rg-nightly-4-22-xqf2d4\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000d504b0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-2mfzr6, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-2mfzr6\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000d504b0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-2mfzr6, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-2mfzr6\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2066657066361032704","occurred_at":"2026-06-15T23:00:15Z","signature_id":"07952c4e0ed6bec072fddb69e111554e9670a77da621835ddbf15edac8169180","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2066657066361032704","occurred_at":"2026-06-15T23:00:15Z","signature_id":"272f9a5fb803702451294089bb1aeaf252e226e80d6a3abc6486bbfc41268e43","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2066657066361032704","occurred_at":"2026-06-15T23:00:15Z","signature_id":"2df063fdb695d32ae24506ffe27697dc68a6bc65cb424655f4f99f3e97592dd9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2066657066361032704","occurred_at":"2026-06-15T23:00:15Z","signature_id":"5cbdbe85570502c86aa4f765ce00a3a49c8475e76a6b778b317a394cdb98291c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2066657066361032704","occurred_at":"2026-06-15T23:00:15Z","signature_id":"8195f6c721d4866295fa5b832468a1b8c1066afa841b7f53dc11319c519ab4c8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2066339954090315776","occurred_at":"2026-06-15T02:00:10Z","signature_id":"2a374a099c3c02f88ba1ec8f04270229deab059fde66652f12efeca7c5d1042b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2066339954090315776","occurred_at":"2026-06-15T02:00:10Z","signature_id":"6b67b09162b9eb5feab8ae82e4573342311a38fab27e1a9e627b6fb22da3ce96","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2066339954090315776","occurred_at":"2026-06-15T02:00:10Z","signature_id":"f5bb9baff8ad5818639da8569f9c1ff49360d72d43f71f8df4d91bfa594252e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2066339954090315776","occurred_at":"2026-06-15T02:00:10Z","signature_id":"f6e60f0c0584835f807e00bfbd250351b4d8bfdd06aee2209d3eab9255ca26fd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-nightly-4-23-x9njsh\" in resourcegroup=\"rg-nightly-4-23-sb2w6f\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-x9njsh in resource group rg-nightly-4-23-sb2w6f, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":4,"runs_affected":2,"failed_runs":2,"run_impact_percent":50,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-06-09..2026-06-15 UTC","prior_windows_present":4,"prior_window_starts":["2026-05-18","2026-05-25","2026-06-01","2026-06-08"],"prior_runs_affected":36,"prior_last_seen_at":"2026-06-14T23:00:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"MISE Routing routes to the correct frontend based on version header default route returns no version header","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-xmg4f9/cluster-nightly-5-0-r9bfms should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000fb4bd0\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-r9bfms\" in resourcegroup=\"rg-nightly-5-0-xmg4f9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-r9bfms in resource group rg-nightly-5-0-xmg4f9, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-5-0-xmg4f9/cluster-nightly-5-0-r9bfms should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000fb4bd0\u003e: \n    failed waiting for cluster=\"cluster-nightly-5-0-r9bfms\" in resourcegroup=\"rg-nightly-5-0-xmg4f9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-5-0-r9bfms in resource group rg-nightly-5-0-xmg4f9, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-sb2w6f/cluster-nightly-4-23-x9njsh should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015723c0\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-x9njsh\" in resourcegroup=\"rg-nightly-4-23-sb2w6f\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-x9njsh in resource group rg-nightly-4-23-sb2w6f, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-sb2w6f/cluster-nightly-4-23-x9njsh should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0015723c0\u003e: \n    failed waiting for cluster=\"cluster-nightly-4-23-x9njsh\" in resourcegroup=\"rg-nightly-4-23-sb2w6f\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-nightly-4-23-x9njsh in resource group rg-nightly-4-23-sb2w6f, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:132]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001462060\u003e: \n    failed waiting for cluster=\"mise-routing\" in resourcegroup=\"mise-default-smoke-mksjs9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster mise-routing in resource group mise-default-smoke-mksjs9, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:132]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001462060\u003e: \n    failed waiting for cluster=\"mise-routing\" in resourcegroup=\"mise-default-smoke-mksjs9\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster mise-routing in resource group mise-default-smoke-mksjs9, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2066657066361032704","occurred_at":"2026-06-15T23:00:15Z","signature_id":"ba30379e687294bff7373300e12d0fbcf9580c2d80a52655c759f51ecc6c9e09","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2066339954090315776","occurred_at":"2026-06-15T02:00:10Z","signature_id":"59cd2f77634741c9174fce5e6b690d0a0e83c99216b75903b6e9c01d033824db","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2066339954090315776","occurred_at":"2026-06-15T02:00:10Z","signature_id":"5dcf5cb128a27c1885f3472aa14eb55533a6ddb2160bb7d14e94706e1cbb0b4d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2066339954090315776","occurred_at":"2026-06-15T02:00:10Z","signature_id":"b01d9f7d9ba4c66db085fb2ff780ebeb6d51c1ee5ed64bf24494b6c41d52b3aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"7426c1a835b553e399edab9b5f3d89e2c53ed02e293edf09d1e41a3543dc8dec","failure_pattern":"ERROR CODE: InvalidTemplateDeployment; provider Microsoft.Compute","search_query":"ERROR CODE: InvalidTemplateDeployment","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":75,"anchor_occurrences":3,"after_last_push_count":8,"trend_counts":[0,0,0,1,2,2,3],"trend_range":"2026-06-09..2026-06-15 UTC","prior_windows_present":1,"prior_window_starts":["2026-06-08"],"prior_runs_affected":8,"prior_last_seen_at":"2026-06-14T23:00:10Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:130]: failed to deploy test VM\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00132d5a0\u003e: \n    failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-pztdvd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-pztdvd/providers/Microsoft.Resources/deployments/test-vm\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplateDeployment\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplateDeployment\",\n        \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.Compute/virtualMachines (2023-03-01)' reported preflight validation errors. Tracking id is 'f6a5d54e-c1c0-4fbe-b749-d7c681bd8016'. See inner errors for details.\",\n        \"details\": [\n          {\n            \"code\": \"QuotaExceeded\",\n            \"message\": \"Operation could not be completed as it results in exceeding approved standardDSv5Family Cores quota. Additional details - Deployment Model: Resource Manager, Location: uksouth, Current Limit: 0, Current Usage: 0, Additional Required: 2, (Minimum) New Limit Required: 2. Setup Alerts when Quota reaches threshold. Learn more at https://aka.ms/quotamonitoringalerting . Submit a request for Quota increase at https://aka.ms/ProdportalCRP/#blade/Microsoft_Azure_Capacity/UsageAndQuota.ReactView/Parameters/%7B%22subscriptionId%22:%22XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%22,%22command%22:%22openQuotaApprovalBlade%22,%22quotas%22:[%7B%22location%22:%22uksouth%22,%22providerId%22:%22Microsoft.Compute%22,%22resourceName%22:%22standardDSv5Family%22,%22quotaRequest%22:%7B%22properties%22:%7B%22limit%22:2,%22unit%22:%22Count%22,%22name%22:%7B%22value%22:%22standardDSv5Family%22%7D%7D%7D%7D]%7D by specifying parameters listed in the ‘Details’ section for deployment to succeed. Please read more about quota limits at https://docs.microsoft.com/en-us/azure/azure-supportability/per-vm-quota-requests\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:130]: failed to deploy test VM\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00132d5a0\u003e: \n    failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-pztdvd\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-pztdvd/providers/Microsoft.Resources/deployments/test-vm\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplateDeployment\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplateDeployment\",\n        \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.Compute/virtualMachines (2023-03-01)' reported preflight validation errors. Tracking id is 'f6a5d54e-c1c0-4fbe-b749-d7c681bd8016'. See inner errors for details.\",\n        \"details\": [\n          {\n            \"code\": \"QuotaExceeded\",\n            \"message\": \"Operation could not be completed as it results in exceeding approved standardDSv5Family Cores quota. Additional details - Deployment Model: Resource Manager, Location: uksouth, Current Limit: 0, Current Usage: 0, Additional Required: 2, (Minimum) New Limit Required: 2. Setup Alerts when Quota reaches threshold. Learn more at https://aka.ms/quotamonitoringalerting . Submit a request for Quota increase at https://aka.ms/ProdportalCRP/#blade/Microsoft_Azure_Capacity/UsageAndQuota.ReactView/Parameters/%7B%22subscriptionId%22:%22XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%22,%22command%22:%22openQuotaApprovalBlade%22,%22quotas%22:[%7B%22location%22:%22uksouth%22,%22providerId%22:%22Microsoft.Compute%22,%22resourceName%22:%22standardDSv5Family%22,%22quotaRequest%22:%7B%22properties%22:%7B%22limit%22:2,%22unit%22:%22Count%22,%22name%22:%7B%22value%22:%22standardDSv5Family%22%7D%7D%7D%7D]%7D by specifying parameters listed in the ‘Details’ section for deployment to succeed. Please read more about quota limits at https://docs.microsoft.com/en-us/azure/azure-supportability/per-vm-quota-requests\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:130]: failed to deploy test VM\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000be6720\u003e: \n    failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-c55hn7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-c55hn7/providers/Microsoft.Resources/deployments/test-vm\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplateDeployment\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplateDeployment\",\n        \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.Compute/virtualMachines (2023-03-01)' reported preflight validation errors. Tracking id is 'c2e9b250-dda1-479b-a710-c7ed5bd06b48'. See inner errors for details.\",\n        \"details\": [\n          {\n            \"code\": \"QuotaExceeded\",\n            \"message\": \"Operation could not be completed as it results in exceeding approved standardDSv5Family Cores quota. Additional details - Deployment Model: Resource Manager, Location: SwitzerlandNorth, Current Limit: 0, Current Usage: 0, Additional Required: 2, (Minimum) New Limit Required: 2. Setup Alerts when Quota reaches threshold. Learn more at https://aka.ms/quotamonitoringalerting . Submit a request for Quota increase at https://aka.ms/ProdportalCRP/#blade/Microsoft_Azure_Capacity/UsageAndQuota.ReactView/Parameters/%7B%22subscriptionId%22:%22XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%22,%22command%22:%22openQuotaApprovalBlade%22,%22quotas%22:[%7B%22location%22:%22SwitzerlandNorth%22,%22providerId%22:%22Microsoft.Compute%22,%22resourceName%22:%22standardDSv5Family%22,%22quotaRequest%22:%7B%22properties%22:%7B%22limit%22:2,%22unit%22:%22Count%22,%22name%22:%7B%22value%22:%22standardDSv5Family%22%7D%7D%7D%7D]%7D by specifying parameters listed in the ‘Details’ section for deployment to succeed. Please read more about quota limits at https://docs.microsoft.com/en-us/azure/azure-supportability/per-vm-quota-requests\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:130]: failed to deploy test VM\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000be6720\u003e: \n    failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-c55hn7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-c55hn7/providers/Microsoft.Resources/deployments/test-vm\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplateDeployment\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplateDeployment\",\n        \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.Compute/virtualMachines (2023-03-01)' reported preflight validation errors. Tracking id is 'c2e9b250-dda1-479b-a710-c7ed5bd06b48'. See inner errors for details.\",\n        \"details\": [\n          {\n            \"code\": \"QuotaExceeded\",\n            \"message\": \"Operation could not be completed as it results in exceeding approved standardDSv5Family Cores quota. Additional details - Deployment Model: Resource Manager, Location: SwitzerlandNorth, Current Limit: 0, Current Usage: 0, Additional Required: 2, (Minimum) New Limit Required: 2. Setup Alerts when Quota reaches threshold. Learn more at https://aka.ms/quotamonitoringalerting . Submit a request for Quota increase at https://aka.ms/ProdportalCRP/#blade/Microsoft_Azure_Capacity/UsageAndQuota.ReactView/Parameters/%7B%22subscriptionId%22:%22XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%22,%22command%22:%22openQuotaApprovalBlade%22,%22quotas%22:[%7B%22location%22:%22SwitzerlandNorth%22,%22providerId%22:%22Microsoft.Compute%22,%22resourceName%22:%22standardDSv5Family%22,%22quotaRequest%22:%7B%22properties%22:%7B%22limit%22:2,%22unit%22:%22Count%22,%22name%22:%7B%22value%22:%22standardDSv5Family%22%7D%7D%7D%7D]%7D by specifying parameters listed in the ‘Details’ section for deployment to succeed. Please read more about quota limits at https://docs.microsoft.com/en-us/azure/azure-supportability/per-vm-quota-requests\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:130]: failed to deploy test VM\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d059c0\u003e: \n    failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-sqzx4f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-sqzx4f/providers/Microsoft.Resources/deployments/test-vm\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplateDeployment\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplateDeployment\",\n        \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.Compute/virtualMachines (2023-03-01)' reported preflight validation errors. Tracking id is '7862bfef-47be-452b-99fa-de6abb702d15'. See inner errors for details.\",\n        \"details\": [\n          {\n            \"code\": \"QuotaExceeded\",\n            \"message\": \"Operation could not be completed as it results in exceeding approved standardDSv5Family Cores quota. Additional details - Deployment Model: Resource Manager, Location: uksouth, Current Limit: 0, Current Usage: 0, Additional Required: 2, (Minimum) New Limit Required: 2. Setup Alerts when Quota reaches threshold. Learn more at https://aka.ms/quotamonitoringalerting . Submit a request for Quota increase at https://aka.ms/ProdportalCRP/#blade/Microsoft_Azure_Capacity/UsageAndQuota.ReactView/Parameters/%7B%22subscriptionId%22:%22XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%22,%22command%22:%22openQuotaApprovalBlade%22,%22quotas%22:[%7B%22location%22:%22uksouth%22,%22providerId%22:%22Microsoft.Compute%22,%22resourceName%22:%22standardDSv5Family%22,%22quotaRequest%22:%7B%22properties%22:%7B%22limit%22:2,%22unit%22:%22Count%22,%22name%22:%7B%22value%22:%22standardDSv5Family%22%7D%7D%7D%7D]%7D by specifying parameters listed in the ‘Details’ section for deployment to succeed. Please read more about quota limits at https://docs.microsoft.com/en-us/azure/azure-supportability/per-vm-quota-requests\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:130]: failed to deploy test VM\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d059c0\u003e: \n    failed creating deployment \"test-vm\" in resourcegroup=\"e2e-cidr-connectivity-sqzx4f\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/e2e-cidr-connectivity-sqzx4f/providers/Microsoft.Resources/deployments/test-vm\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidTemplateDeployment\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidTemplateDeployment\",\n        \"message\": \"The template deployment 'test-vm' is not valid according to the validation procedure. The following resource provider(s) - 'Microsoft.Compute/virtualMachines (2023-03-01)' reported preflight validation errors. Tracking id is '7862bfef-47be-452b-99fa-de6abb702d15'. See inner errors for details.\",\n        \"details\": [\n          {\n            \"code\": \"QuotaExceeded\",\n            \"message\": \"Operation could not be completed as it results in exceeding approved standardDSv5Family Cores quota. Additional details - Deployment Model: Resource Manager, Location: uksouth, Current Limit: 0, Current Usage: 0, Additional Required: 2, (Minimum) New Limit Required: 2. Setup Alerts when Quota reaches threshold. Learn more at https://aka.ms/quotamonitoringalerting . Submit a request for Quota increase at https://aka.ms/ProdportalCRP/#blade/Microsoft_Azure_Capacity/UsageAndQuota.ReactView/Parameters/%7B%22subscriptionId%22:%22XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%22,%22command%22:%22openQuotaApprovalBlade%22,%22quotas%22:[%7B%22location%22:%22uksouth%22,%22providerId%22:%22Microsoft.Compute%22,%22resourceName%22:%22standardDSv5Family%22,%22quotaRequest%22:%7B%22properties%22:%7B%22limit%22:2,%22unit%22:%22Count%22,%22name%22:%7B%22value%22:%22standardDSv5Family%22%7D%7D%7D%7D]%7D by specifying parameters listed in the ‘Details’ section for deployment to succeed. Please read more about quota limits at https://docs.microsoft.com/en-us/azure/azure-supportability/per-vm-quota-requests\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2066657066361032704","occurred_at":"2026-06-15T23:00:15Z","signature_id":"2ff0503dae16648b4872fdaa69472c6a46b318ce3a79d26505e21420b1095b62","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2066611850006499328","occurred_at":"2026-06-15T20:00:19Z","signature_id":"3bffcf83322021fbaca59a677c586af6968ea68a5fbda0b59c0c8c98a7758e10","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2066339954090315776","occurred_at":"2026-06-15T02:00:10Z","signature_id":"ad4694f9b254147813498a853e6e8d3dfe0f1296f7b5b5c1d920d30cf216fbfc","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"ad09ec798e62ff2c2e0efff8b4d734e50bd5198a307cc9f3128e29ec002eb45a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Version 'openshift-v4.23.0-0.nightly-multi-2026-06-15-224233-nightly' doesn't exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-06-09..2026-06-15 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-jhhdtf/cluster-nightly-4-23-qw58nx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006afaa0\u003e: \n    failed starting cluster creation \"cluster-nightly-4-23-qw58nx\" in resourcegroup=\"rg-nightly-4-23-jhhdtf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-23-jhhdtf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-23-qw58nx\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.23.0-0.nightly-multi-2026-06-15-224233-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:126]: Swift cluster rg-nightly-4-23-jhhdtf/cluster-nightly-4-23-qw58nx should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0006afaa0\u003e: \n    failed starting cluster creation \"cluster-nightly-4-23-qw58nx\" in resourcegroup=\"rg-nightly-4-23-jhhdtf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-23-jhhdtf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-23-qw58nx\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Version 'openshift-v4.23.0-0.nightly-multi-2026-06-15-224233-nightly' doesn't exist\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2066657066361032704","occurred_at":"2026-06-15T23:00:15Z","signature_id":"5049f31d90331b27543d536839338fb93be1429d7d69ff5fb8a9acb53afd82b1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,2,0,1],"trend_range":"2026-06-09..2026-06-15 UTC","prior_windows_present":2,"prior_window_starts":["2026-05-18","2026-06-08"],"prior_runs_affected":5,"prior_last_seen_at":"2026-06-13T23:00:09Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0009bc420\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-nsz4km, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-nsz4km\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:190]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0009bc420\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool cilium-np in resource group complex-cilium-kv-nsz4km, error: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-nsz4km\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2066339954090315776","occurred_at":"2026-06-15T02:00:10Z","signature_id":"ad9297c2646cd31cfa64e00b5336e8eabf3a8efc2fdb992f7e18ffd59bcc60f6","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}