{"meta":{"start_date":"2026-07-13","end_date":"2026-07-13","start_at":"2026-07-13T00:00:00Z","end_at":"2026-07-14T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T21:16:39Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":2,"failed_runs":2,"raw_occurrences":4,"matched_occurrences":4,"runs_affected":2},"rows":[{"environment":"stg","failure_pattern_id":"6ed54686dd2f6d32054cbf144906e399a60d9c8b1a6d87d91850a99759063298","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster cluster-candidate-4-21-hv9d74, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-hv9d74 in resource group rg-candidate-4-21-r7bcpn, error: failed waiting for cluster=\"cluster-candidate-4-21-hv9d74\" in resourcegroup=\"rg-candidate-4-21-r7bcpn\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-hv9d74 in resource group rg-candidate-4-21-r7bcpn, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":100,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-07-07..2026-07-13 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-15","2026-06-22","2026-06-29","2026-07-06"],"prior_runs_affected":54,"prior_last_seen_at":"2026-07-10T16:57:45Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-5-0-bn4lfp/cluster-nightly-5-0-9z27s2 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013be6f0\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-9z27s2, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-9z27s2 in resource group rg-nightly-5-0-bn4lfp, error: failed waiting for cluster=\"cluster-nightly-5-0-9z27s2\" in resourcegroup=\"rg-nightly-5-0-bn4lfp\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-9z27s2 in resource group rg-nightly-5-0-bn4lfp, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-nightly-5-0-bn4lfp/cluster-nightly-5-0-9z27s2 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013be6f0\u003e: \n    failed to create HCP cluster cluster-nightly-5-0-9z27s2, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-9z27s2 in resource group rg-nightly-5-0-bn4lfp, error: failed waiting for cluster=\"cluster-nightly-5-0-9z27s2\" in resourcegroup=\"rg-nightly-5-0-bn4lfp\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-nightly-5-0-9z27s2 in resource group rg-nightly-5-0-bn4lfp, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-21-r7bcpn/cluster-candidate-4-21-hv9d74 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000bdb860\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-hv9d74, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-hv9d74 in resource group rg-candidate-4-21-r7bcpn, error: failed waiting for cluster=\"cluster-candidate-4-21-hv9d74\" in resourcegroup=\"rg-candidate-4-21-r7bcpn\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-hv9d74 in resource group rg-candidate-4-21-r7bcpn, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:106]: HCP cluster rg-candidate-4-21-r7bcpn/cluster-candidate-4-21-hv9d74 should provision\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000bdb860\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-hv9d74, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-hv9d74 in resource group rg-candidate-4-21-r7bcpn, error: failed waiting for cluster=\"cluster-candidate-4-21-hv9d74\" in resourcegroup=\"rg-candidate-4-21-r7bcpn\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster cluster-candidate-4-21-hv9d74 in resource group rg-candidate-4-21-r7bcpn, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2076804081216655360","occurred_at":"2026-07-13T23:00:52Z","signature_id":"e7320526ee2f0e163d5167184a235a1781167d853d98857967c02d0d429ae64f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2076486900469731328","occurred_at":"2026-07-13T02:00:30Z","signature_id":"046fbb58b1bfa2d126ed95bec55b92c547025ffd9bab3885118a2c3ed811874a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"4e83fe1c3bb6daff1be6935129ccd1a1c67e021531f7a464980385c362ccc372","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-pch5q2\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-pch5q2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-pch5q2, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-pch5q2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-pch5q2, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-07..2026-07-13 UTC","prior_windows_present":1,"prior_window_starts":["2026-07-06"],"prior_runs_affected":4,"prior_last_seen_at":"2026-07-12T00:45:19Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc000fd82a0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-pch5q2\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-pch5q2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-pch5q2, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-pch5q2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-pch5q2, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc000fd82a0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-pch5q2\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-pch5q2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-pch5q2, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-pch5q2\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-pch5q2, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2076486900469731328","occurred_at":"2026-07-13T02:00:30Z","signature_id":"ee4ac2344972f911116892bb11b81eafaa1d132790b8d4a50f21523c5df86d7b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"3bfb93ffa2915664d7de20fc0b753c1a5307e53691a4b8098db3a7e206ca4906","failure_pattern":"x509: certificate signed by unknown authority","search_query":"x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":50,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-07..2026-07-13 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:167]: Timed out after 600.000s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:167]: Timed out after 600.000s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2076804081216655360","occurred_at":"2026-07-13T23:00:52Z","signature_id":"20cd25cfb84069bf0dbeae90ccfea853127cb3d7c1e8d8f537a39512ce31cc19","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}