{"meta":{"start_date":"2026-07-11","end_date":"2026-07-11","start_at":"2026-07-11T00:00:00Z","end_at":"2026-07-12T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T06:14:34Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":5,"failed_runs":3,"raw_occurrences":6,"matched_occurrences":6,"runs_affected":3},"rows":[{"environment":"int","failure_pattern_id":"f90cae3228fe72b2427515e7c51834eebdc3e73c0b1aa6bca2abedd53a2e7578","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"ephemeral-osdisk-wdqlbm\", caused by: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-wdqlbm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-wdqlbm, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-wdqlbm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-wdqlbm, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":40,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-07-05..2026-07-11 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc00199a1f8\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-zstream-upgrade-4-22-xddbdf\", caused by: failed waiting for hcpCluster=\"cluster-zstream-4-22-qz9tbp\" in resourcegroup=\"rg-zstream-upgrade-4-22-xddbdf\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-zstream-4-22-qz9tbp in resource group rg-zstream-upgrade-4-22-xddbdf, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-zstream-4-22-qz9tbp\" in resourcegroup=\"rg-zstream-upgrade-4-22-xddbdf\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-zstream-4-22-qz9tbp in resource group rg-zstream-upgrade-4-22-xddbdf, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc00199a1f8\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-zstream-upgrade-4-22-xddbdf\", caused by: failed waiting for hcpCluster=\"cluster-zstream-4-22-qz9tbp\" in resourcegroup=\"rg-zstream-upgrade-4-22-xddbdf\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-zstream-4-22-qz9tbp in resource group rg-zstream-upgrade-4-22-xddbdf, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-zstream-4-22-qz9tbp\" in resourcegroup=\"rg-zstream-upgrade-4-22-xddbdf\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-zstream-4-22-qz9tbp in resource group rg-zstream-upgrade-4-22-xddbdf, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc00125a0f0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"ephemeral-osdisk-wdqlbm\", caused by: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-wdqlbm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-wdqlbm, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-wdqlbm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-wdqlbm, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc00125a0f0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"ephemeral-osdisk-wdqlbm\", caused by: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-wdqlbm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-wdqlbm, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-wdqlbm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-wdqlbm, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc000538648\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-pndqms\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-pndqms\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-pndqms, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-pndqms\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-pndqms, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc000538648\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-pndqms\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-pndqms\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-pndqms, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-pndqms\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-pndqms, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2075795979096821760","occurred_at":"2026-07-11T04:14:57Z","signature_id":"0bbfece7a73d1d79ec9fa7b246d1331b2fa1e8784b0f7f42261ef3ef5d893edf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2075795979096821760","occurred_at":"2026-07-11T04:14:57Z","signature_id":"90699d43e46d95701ad6ea828fe60e19503360d6286a852fceccd8a0e1152ae0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2075737788124237824","occurred_at":"2026-07-11T00:23:48Z","signature_id":"8ec225040ae000d45a9c881d774dfb2b781dffc5ea704607532b22a5019ccceb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"a880b7dbbe8686372b606e0212fd3742bc75c3c0a9c545bae9481b243ed0f55b","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-7rhzs8 in resource group rg-candidate-4-21-c5mrcr, error: failed waiting for nodepool=\"np-7rhzs8\" for cluster \"cluster-candidate-4-21-7rhzs8\" in resourcegroup=\"rg-candidate-4-21-c5mrcr\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-05..2026-07-11 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-08","2026-06-15","2026-06-22","2026-06-29"],"prior_runs_affected":18,"prior_last_seen_at":"2026-07-03T00:49:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-7rhzs8\" for cluster \"cluster-candidate-4-21-7rhzs8\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00278eba0\u003e: \n    failed to create NodePool np-7rhzs8, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-7rhzs8 in resource group rg-candidate-4-21-c5mrcr, error: failed waiting for nodepool=\"np-7rhzs8\" for cluster \"cluster-candidate-4-21-7rhzs8\" in resourcegroup=\"rg-candidate-4-21-c5mrcr\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-7rhzs8\" for cluster \"cluster-candidate-4-21-7rhzs8\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00278eba0\u003e: \n    failed to create NodePool np-7rhzs8, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-7rhzs8 in resource group rg-candidate-4-21-c5mrcr, error: failed waiting for nodepool=\"np-7rhzs8\" for cluster \"cluster-candidate-4-21-7rhzs8\" in resourcegroup=\"rg-candidate-4-21-c5mrcr\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2075795979096821760","occurred_at":"2026-07-11T04:14:57Z","signature_id":"a52cdb4c37ecd3e0bb01e423d1bb47f929c074c9ffd32946a6f3fa7d79c29bb1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"0ece011306dab74cd22dd19f1f3c89bbdb930a2321b61be59e4a6a90aae6e3eb","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-downgrade-g9g6xf in resource group rg-np-downgrade-g9g6xf-875q59, error: failed waiting for cluster=\"np-downgrade-g9g6xf\" in resourcegroup=\"rg-np-downgrade-g9g6xf-875q59\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-downgrade-g9g6xf in resource group rg-np-downgrade-g9g6xf-875q59, error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-05..2026-07-11 UTC","prior_windows_present":4,"prior_window_starts":["2026-06-08","2026-06-15","2026-06-22","2026-06-29"],"prior_runs_affected":32,"prior_last_seen_at":"2026-07-03T09:28:40Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:589]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00095c9f0\u003e: \n    failed to create HCP cluster np-downgrade-g9g6xf, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-downgrade-g9g6xf in resource group rg-np-downgrade-g9g6xf-875q59, error: failed waiting for cluster=\"np-downgrade-g9g6xf\" in resourcegroup=\"rg-np-downgrade-g9g6xf-875q59\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-downgrade-g9g6xf in resource group rg-np-downgrade-g9g6xf-875q59, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:589]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00095c9f0\u003e: \n    failed to create HCP cluster np-downgrade-g9g6xf, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-downgrade-g9g6xf in resource group rg-np-downgrade-g9g6xf-875q59, error: failed waiting for cluster=\"np-downgrade-g9g6xf\" in resourcegroup=\"rg-np-downgrade-g9g6xf-875q59\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-downgrade-g9g6xf in resource group rg-np-downgrade-g9g6xf-875q59, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2075795979096821760","occurred_at":"2026-07-11T04:14:57Z","signature_id":"3ecd07f0a12870c530dfd4ed341fae9dc43b6dd272631f771d92be5c322b9605","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"98e32667c3e390e96a18d1f5c467888c22b014b872e3aaaa2bfc4f15efac6c45","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-07-05..2026-07-11 UTC","prior_windows_present":2,"prior_window_starts":["2026-06-15","2026-06-29"],"prior_runs_affected":3,"prior_last_seen_at":"2026-07-01T09:36:59Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:194]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc00131c1b0\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-x5slv.apps.aro.cilium-cluster.f5i8.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:194]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc00131c1b0\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-x5slv.apps.aro.cilium-cluster.f5i8.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2075874054463557632","occurred_at":"2026-07-11T09:25:17Z","signature_id":"b99b478f99e0d9fd2bcfa35f8c4fcccffcc688449db29d2fa9239e783b70a678","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}