{"meta":{"start_date":"2026-06-23","end_date":"2026-06-23","start_at":"2026-06-23T00:00:00Z","end_at":"2026-06-24T00:00:00Z","timezone":"UTC","generated_at":"2026-08-27T00:50:19Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":6,"failed_runs":5,"raw_occurrences":14,"matched_occurrences":14,"runs_affected":5},"rows":[{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-20-kxcsr7, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-kxcsr7 in resource group rg-candidate-4-20-79p2x6, error: failed waiting for cluster=\"cluster-candidate-4-20-kxcsr7\" in resourcegroup=\"rg-candidate-4-20-79p2x6\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-20-kxcsr7 in resource group rg-candidate-4-20-79p2x6, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":12,"runs_affected":5,"failed_runs":5,"run_impact_percent":83.33333333333333,"anchor_occurrences":12,"after_last_push_count":12,"trend_counts":[0,0,0,0,0,0,12],"trend_range":"2026-06-17..2026-06-23 UTC","prior_windows_present":4,"prior_window_starts":["2026-05-25","2026-06-01","2026-06-08","2026-06-15"],"prior_runs_affected":41,"prior_last_seen_at":"2026-06-20T23:00:27Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000abb3e0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-vmskgh\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-vmskgh, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:97]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000abb3e0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-vmskgh\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-vmskgh, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012ec840\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-2prfsk, error: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-2prfsk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-2prfsk, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:94]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0012ec840\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-2prfsk, error: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-2prfsk\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster tls-endpoint-hcp-cluster in resource group tls-endpoint-cluster-2prfsk, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-4-23-spgzxc/cluster-nightly-4-23-7drz4x should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000c678f0\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-7drz4x, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-7drz4x in resource group rg-nightly-4-23-spgzxc, error: failed waiting for cluster=\"cluster-nightly-4-23-7drz4x\" in resourcegroup=\"rg-nightly-4-23-spgzxc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-7drz4x in resource group rg-nightly-4-23-spgzxc, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-4-23-spgzxc/cluster-nightly-4-23-7drz4x should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000c678f0\u003e: \n    failed to create HCP cluster cluster-nightly-4-23-7drz4x, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-7drz4x in resource group rg-nightly-4-23-spgzxc, error: failed waiting for cluster=\"cluster-nightly-4-23-7drz4x\" in resourcegroup=\"rg-nightly-4-23-spgzxc\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-4-23-7drz4x in resource group rg-nightly-4-23-spgzxc, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2069556130496909312","occurred_at":"2026-06-23T23:00:06Z","signature_id":"08dde14e0755e6ea5b06fb014131678e7fc05e185482f637b523df0497a0d17d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2069556130496909312","occurred_at":"2026-06-23T23:00:06Z","signature_id":"2582df8cbd124d0d4a2afb9fba60b204fc53a7e20cd4a767a81f1dc6d5a109ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2069556130496909312","occurred_at":"2026-06-23T23:00:06Z","signature_id":"46d389d69ba42fef1cc8ad1bb6252c4b6f97d5695dee2b1efd6a91ebb364a6c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2069556130496909312","occurred_at":"2026-06-23T23:00:06Z","signature_id":"9d2e37a41ac31e236364c0b5b397cde40ae39ceb2e35d9503711f98fb79c863b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2069556130496909312","occurred_at":"2026-06-23T23:00:06Z","signature_id":"b1be0ad549d21afa914b5d035cac2ffad0abaea30acb1bd837d520abc999f275","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2069556130496909312","occurred_at":"2026-06-23T23:00:06Z","signature_id":"c498bbc4ca442add49dd1958a514c1156eebc818c8194225bed803c7d97580ff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2069556130496909312","occurred_at":"2026-06-23T23:00:06Z","signature_id":"ca448ffc21aca0ff90acd1329ed31e2286331f478238276c000031e6a14ac104","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2069417000056131584","occurred_at":"2026-06-23T13:46:50Z","signature_id":"2a2d884f4501c77edc8bcbde4a690b46b876202da711d3c46da82c5214c89ec7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2069264571155091456","occurred_at":"2026-06-23T03:41:33Z","signature_id":"2008f8bdf7e6770f61faf0871a346f51f3345fcfb4ed32d1cdfaa7d13d267976","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2069264571155091456","occurred_at":"2026-06-23T03:41:33Z","signature_id":"3946fab07c0358b15d5ea4fab6fc2928896b66fabf448208ac28abdef2f554c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2069239037184446464","occurred_at":"2026-06-23T02:00:05Z","signature_id":"6dc6abb6d49682cf350aed62faed9312f2b8be133716fe5adebe992063546d53","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2069230875538624512","occurred_at":"2026-06-23T01:27:39Z","signature_id":"ca66aa298134cfa257dfb7f99c1e390a42943ce646f440e0820b5ca0b7d2aad1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"941cbd5c6e3f3d8ed572893a689b2dfded75f418444a1a0d01baa9fc9e30d9c0","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-6h8b6r in resource group rg-candidate-4-21-2bqcn5, error: failed waiting for nodepool=\"np-6h8b6r\" for cluster \"cluster-candidate-4-21-6h8b6r\" in resourcegroup=\"rg-candidate-4-21-2bqcn5\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-06-17..2026-06-23 UTC","prior_windows_present":3,"prior_window_starts":["2026-05-25","2026-06-08","2026-06-15"],"prior_runs_affected":15,"prior_last_seen_at":"2026-06-19T19:37:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-6h8b6r\" for cluster \"cluster-candidate-4-21-6h8b6r\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001537c80\u003e: \n    failed to create NodePool np-6h8b6r, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-6h8b6r in resource group rg-candidate-4-21-2bqcn5, error: failed waiting for nodepool=\"np-6h8b6r\" for cluster \"cluster-candidate-4-21-6h8b6r\" in resourcegroup=\"rg-candidate-4-21-2bqcn5\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-6h8b6r\" for cluster \"cluster-candidate-4-21-6h8b6r\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc001537c80\u003e: \n    failed to create NodePool np-6h8b6r, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-6h8b6r in resource group rg-candidate-4-21-2bqcn5, error: failed waiting for nodepool=\"np-6h8b6r\" for cluster \"cluster-candidate-4-21-6h8b6r\" in resourcegroup=\"rg-candidate-4-21-2bqcn5\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2069417000056131584","occurred_at":"2026-06-23T13:46:50Z","signature_id":"9d41f5140c8f9c893570641b224ac88b48ff81c1c5266bea4d485a0498d7d8ff","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7b7295e8e9922f80f23efae98869be37ffdadd068d7c16232ad6eaf824f752ad","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-06-17..2026-06-23 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:170]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000881980\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-qfrzh.apps.aro.cilium-cl.9zjy.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:170]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc000881980\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-qfrzh.apps.aro.cilium-cl.9zjy.uksouthstaging.aroapp-hcp.io\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2069230875538624512","occurred_at":"2026-06-23T01:27:39Z","signature_id":"c7d4b42baef0d672054b5dbb2d77c68db076879abc452803eeb38b332bb67ec5","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}