{"meta":{"start_date":"2026-05-22","end_date":"2026-05-22","start_at":"2026-05-22T00:00:00Z","end_at":"2026-05-23T00:00:00Z","timezone":"UTC","generated_at":"2026-08-27T08:07:35Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":1,"failed_runs":1,"raw_occurrences":10,"matched_occurrences":10,"runs_affected":1},"rows":[{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-dcr2kv\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group complex-cilium-kv-dcr2kv, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":8,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":8,"after_last_push_count":8,"trend_counts":[0,0,0,0,0,0,8],"trend_range":"2026-05-16..2026-05-22 UTC","prior_windows_present":4,"prior_window_starts":["2026-04-20","2026-04-27","2026-05-04","2026-05-11"],"prior_runs_affected":14,"prior_last_seen_at":"2026-05-13T02:00:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: Swift cluster rg-candidate-5-0-7jt2zr/cluster-candidate-5-0-898h7z should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a159e0\u003e: \n    failed waiting for cluster=\"cluster-candidate-5-0-898h7z\" in resourcegroup=\"rg-candidate-5-0-7jt2zr\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-candidate-5-0-898h7z in resource group rg-candidate-5-0-7jt2zr, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:120]: Swift cluster rg-candidate-5-0-7jt2zr/cluster-candidate-5-0-898h7z should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000a159e0\u003e: \n    failed waiting for cluster=\"cluster-candidate-5-0-898h7z\" in resourcegroup=\"rg-candidate-5-0-7jt2zr\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cluster-candidate-5-0-898h7z in resource group rg-candidate-5-0-7jt2zr, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137]: failed to create HCP cluster \"cluster-zstream-4-22-9dtk7p\" with version 4.22.0-rc.2 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00114e9f0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-9dtk7p, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-9dtk7p in resource group rg-zstream-upgrade-4-22-68pdm8, error: failed waiting for cluster=\"cluster-zstream-4-22-9dtk7p\" in resourcegroup=\"rg-zstream-upgrade-4-22-68pdm8\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-9dtk7p in resource group rg-zstream-upgrade-4-22-68pdm8, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137]: failed to create HCP cluster \"cluster-zstream-4-22-9dtk7p\" with version 4.22.0-rc.2 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00114e9f0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-9dtk7p, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-9dtk7p in resource group rg-zstream-upgrade-4-22-68pdm8, error: failed waiting for cluster=\"cluster-zstream-4-22-9dtk7p\" in resourcegroup=\"rg-zstream-upgrade-4-22-68pdm8\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-22-9dtk7p in resource group rg-zstream-upgrade-4-22-68pdm8, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:108]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000c45a70\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-frs6rt\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-frs6rt, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:108]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000c45a70\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-frs6rt\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-frs6rt, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2057642836106416128","occurred_at":"2026-05-22T02:00:55Z","signature_id":"577d1a8631ed807a129f173b353d0c400d8fc6186409656ccb0153e6099ba182","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2057642836106416128","occurred_at":"2026-05-22T02:00:55Z","signature_id":"94e7c86db7db97773b4d5fb1f2ac3d3ec879efde31a1f16adf39d48afd98a3b1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2057642836106416128","occurred_at":"2026-05-22T02:00:55Z","signature_id":"956a78ae05e45dc10d608917d9ae61f1a5a656233dbb463d9a425e47a983b4a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2057642836106416128","occurred_at":"2026-05-22T02:00:55Z","signature_id":"9d85d4c40d0883d8af17bf5f9ab8c4685dcd2ff4cd3fca940e7f1044b77d092d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2057642836106416128","occurred_at":"2026-05-22T02:00:55Z","signature_id":"a015bcbe25caf8c94d40e4903cd8326b06aa1fd1d518ae10fba255588b182f3f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2057642836106416128","occurred_at":"2026-05-22T02:00:55Z","signature_id":"a1ac8cebe05dc51e9d4f3cbcf97ff66d42c003d27cc08f9261f604325879b187","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2057642836106416128","occurred_at":"2026-05-22T02:00:55Z","signature_id":"c4e0f0d8f42689b7d3157bea0df1ab8dab1904a9d85d7a2c4cf6993d818161fd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2057642836106416128","occurred_at":"2026-05-22T02:00:55Z","signature_id":"f680d403ed6699312938f386d7a3b54417a74206875686f2c28b71d8b4d07ceb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"6a192df6a0bf1b1deb955be87b396535fa194d37776b6605a1e34858e7266e88","failure_pattern":"failed to wait for first cluster \"\u003ccluster\u003e\" to complete creation","search_query":"context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-05-16..2026-05-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:132]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:132]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation\nUnexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2057642836106416128","occurred_at":"2026-05-22T02:00:55Z","signature_id":"3e3a3d4f62c5e8a64ff8b5cb02b93498aaf8a6753c80e2b6b8b3864addabc2a3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"d930e05681864b995fbc671ac47fdf78f3f93177c4bbe97b6c62da858d21cf69","failure_pattern":"validatingadmissionpolicies.admissionregistration.k8s.io \"image-registry-allowlist-policy\" is forbidden: User \"system:serviceaccount:ci-op-n12p2cds:stage-e2e-parallel\" cannot get resource \"validatingadmissionpolicies\" in API group","search_query":"validatingadmissionpolicies.admissionregistration.k8s.io \"image-registry-allowlist-policy\" is forbidden: User \"system:serviceaccount:ci-op-n12p2cds:stage-e2e-parallel\" cannot get resource \"validatingadmissionpolicies\" in API group \"admissionregistration.k8s.io\" at the cluster scope","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Image Registry Policy should allow pods with images from allowed registries and have a valid allowlist","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-05-16..2026-05-22 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Image Registry Policy should allow pods with images from allowed registries and have a valid allowlist","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_policy.go:135]: ValidatingAdmissionPolicy should exist\nUnexpected error:\n    \u003c*errors.StatusError | 0xc000e0e140\u003e: \n    validatingadmissionpolicies.admissionregistration.k8s.io \"image-registry-allowlist-policy\" is forbidden: User \"system:serviceaccount:ci-op-n12p2cds:stage-e2e-parallel\" cannot get resource \"validatingadmissionpolicies\" in API group \"admissionregistration.k8s.io\" at the cluster scope\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_policy.go:135]: ValidatingAdmissionPolicy should exist\nUnexpected error:\n    \u003c*errors.StatusError | 0xc000e0e140\u003e: \n    validatingadmissionpolicies.admissionregistration.k8s.io \"image-registry-allowlist-policy\" is forbidden: User \"system:serviceaccount:ci-op-n12p2cds:stage-e2e-parallel\" cannot get resource \"validatingadmissionpolicies\" in API group \"admissionregistration.k8s.io\" at the cluster scope\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2057642836106416128","occurred_at":"2026-05-22T02:00:55Z","signature_id":"39805d4c7a79c157d322c03a96fc601e9170b64e63caabb13b03a6dc5e8345ce","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}