{"meta":{"start_date":"2026-08-17","end_date":"2026-08-17","start_at":"2026-08-17T00:00:00Z","end_at":"2026-08-18T00:00:00Z","timezone":"UTC","generated_at":"2026-08-25T23:08:08Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":2,"failed_runs":2,"raw_occurrences":4,"matched_occurrences":4,"runs_affected":2},"rows":[{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-nightly-5-1-772lzj, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-772lzj in resource group rg-nightly-5-1-bdzmnlk6jx8p, error: failed waiting for cluster=\"cluster-nightly-5-1-772lzj\" in resourcegroup=\"rg-nightly-5-1-bdzmnlk6jx8p\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-772lzj in resource group rg-nightly-5-1-bdzmnlk6jx8p, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-08-11..2026-08-17 UTC","prior_windows_present":4,"prior_window_starts":["2026-07-20","2026-07-27","2026-08-03","2026-08-10"],"prior_runs_affected":21,"prior_last_seen_at":"2026-08-16T23:00:08Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002bf0450\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-dq2lkrsd2vbw, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-dq2lkrsd2vbw\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-dq2lkrsd2vbw, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002bf0450\u003e: \n    failed to create HCP cluster private-kas, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-dq2lkrsd2vbw, error: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-dq2lkrsd2vbw\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster private-kas in resource group private-kas-dq2lkrsd2vbw, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-1-bdzmnlk6jx8p/cluster-nightly-5-1-772lzj should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000aaeb70\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-772lzj, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-772lzj in resource group rg-nightly-5-1-bdzmnlk6jx8p, error: failed waiting for cluster=\"cluster-nightly-5-1-772lzj\" in resourcegroup=\"rg-nightly-5-1-bdzmnlk6jx8p\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-772lzj in resource group rg-nightly-5-1-bdzmnlk6jx8p, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:107]: HCP cluster rg-nightly-5-1-bdzmnlk6jx8p/cluster-nightly-5-1-772lzj should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000aaeb70\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-772lzj, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-772lzj in resource group rg-nightly-5-1-bdzmnlk6jx8p, error: failed waiting for cluster=\"cluster-nightly-5-1-772lzj\" in resourcegroup=\"rg-nightly-5-1-bdzmnlk6jx8p\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-1-772lzj in resource group rg-nightly-5-1-bdzmnlk6jx8p, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2089487626737291264","occurred_at":"2026-08-17T23:00:45Z","signature_id":"05561418fc69f3cca55dfbb58d3323b77c49d65368b118a5744259edf619bb45","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2089487626737291264","occurred_at":"2026-08-17T23:00:45Z","signature_id":"095760629591549f0f071776b5e5de49a0cdfef888cc7ee0bb080f9444008b13","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"933b94411053a9d41a8fd445af49db17270759cb545630b19bb968f9dc15ecc1","failure_pattern":"VerifyVMOSDiskCustomerEncryption(nodePool=\u003cnodepool\u003e) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-wwjjh-j45jb_OSDisk for VM des-encrypt-des-np-wwjjh-j45jb has encryption type EncryptionAtRestWithPlatformKey, expected","search_query":"VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-wwjjh-j45jb_OSDisk for VM des-encrypt-des-np-wwjjh-j45jb has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-11..2026-08-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:163]: cluster verification failed\nUnexpected error:\n    \u003c*errors.joinError | 0xc001c9d968\u003e: \n    VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-wwjjh-j45jb_OSDisk for VM des-encrypt-des-np-wwjjh-j45jb has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    disk des-encrypt-des-np-wwjjh-pclnh_OSDisk for VM des-encrypt-des-np-wwjjh-pclnh has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:163]: cluster verification failed\nUnexpected error:\n    \u003c*errors.joinError | 0xc001c9d968\u003e: \n    VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-wwjjh-j45jb_OSDisk for VM des-encrypt-des-np-wwjjh-j45jb has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    disk des-encrypt-des-np-wwjjh-pclnh_OSDisk for VM des-encrypt-des-np-wwjjh-pclnh has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2089487626737291264","occurred_at":"2026-08-17T23:00:45Z","signature_id":"dfef87c3d3473ee46b192d27d4af2c584e0740ceb219a88833f2f2072dc384ef","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"040ccb69166de13bd0ff178d2b746793e77ee46272b69835afa7aabd3e8c3b82","failure_pattern":"VerifyVMOSDiskCustomerEncryption(nodePool=\u003cnodepool\u003e) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-zbqjh-d2z8p_OSDisk for VM des-encrypt-des-np-zbqjh-d2z8p has encryption type EncryptionAtRestWithPlatformKey, expected","search_query":"VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-zbqjh-d2z8p_OSDisk for VM des-encrypt-des-np-zbqjh-d2z8p has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-08-11..2026-08-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool OS Disk Encryption should create a nodepool with customer-managed disk encryption via DES","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:163]: cluster verification failed\nUnexpected error:\n    \u003c*errors.joinError | 0xc000f7a858\u003e: \n    VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-zbqjh-d2z8p_OSDisk for VM des-encrypt-des-np-zbqjh-d2z8p has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    disk des-encrypt-des-np-zbqjh-hb6cr_OSDisk for VM des-encrypt-des-np-zbqjh-hb6cr has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_osdisk_encryption.go:163]: cluster verification failed\nUnexpected error:\n    \u003c*errors.joinError | 0xc000f7a858\u003e: \n    VerifyVMOSDiskCustomerEncryption(nodePool=des-np) failed: OS disk encryption verification failed for 2/2 VMs: disk des-encrypt-des-np-zbqjh-d2z8p_OSDisk for VM des-encrypt-des-np-zbqjh-d2z8p has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    disk des-encrypt-des-np-zbqjh-hb6cr_OSDisk for VM des-encrypt-des-np-zbqjh-hb6cr has encryption type EncryptionAtRestWithPlatformKey, expected EncryptionAtRestWithCustomerKey\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2089170383671398400","occurred_at":"2026-08-17T02:00:08Z","signature_id":"ef5ba68c8950e27b3f885e7fe494453e583a9c1347443e1d8b07670cfb5833b3","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}