{"meta":{"date":"2026-07-11","timezone":"UTC","generated_at":"2026-08-27T20:57:40Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":5,"failed_runs":3,"runs_with_occurrences":3,"runs_with_failure_pattern_matches":3,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":2,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2076058892294426624","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-11T21:38:27Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2076008155677986816","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-11T18:18:09Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2075874054463557632","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-11T09:25:17Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"1f2d7dfbd2e5bdfe93d66b68bb37986764e0271f40f166cb4fb204ac46aca234","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2075874054463557632","occurred_at":"2026-07-11T09:25:17Z","failed_at":"e2e","signature_id":"b99b478f99e0d9fd2bcfa35f8c4fcccffcc688449db29d2fa9239e783b70a678","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:194]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc00131c1b0\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-x5slv.apps.aro.cilium-cluster.f5i8.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:194]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc00131c1b0\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-x5slv.apps.aro.cilium-cluster.f5i8.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98e32667c3e390e96a18d1f5c467888c22b014b872e3aaaa2bfc4f15efac6c45","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["98e32667c3e390e96a18d1f5c467888c22b014b872e3aaaa2bfc4f15efac6c45"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2075795979096821760","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-11T04:14:57Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"33368300abd298a47c9a7c4ad4c45fc1aeed2a4a4b864da95d6510dd4cbcf205","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2075795979096821760","occurred_at":"2026-07-11T04:14:57Z","failed_at":"e2e","signature_id":"3ecd07f0a12870c530dfd4ed341fae9dc43b6dd272631f771d92be5c322b9605","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:589]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00095c9f0\u003e: \n    failed to create HCP cluster np-downgrade-g9g6xf, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-downgrade-g9g6xf in resource group rg-np-downgrade-g9g6xf-875q59, error: failed waiting for cluster=\"np-downgrade-g9g6xf\" in resourcegroup=\"rg-np-downgrade-g9g6xf-875q59\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-downgrade-g9g6xf in resource group rg-np-downgrade-g9g6xf-875q59, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:589]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00095c9f0\u003e: \n    failed to create HCP cluster np-downgrade-g9g6xf, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-downgrade-g9g6xf in resource group rg-np-downgrade-g9g6xf-875q59, error: failed waiting for cluster=\"np-downgrade-g9g6xf\" in resourcegroup=\"rg-np-downgrade-g9g6xf-875q59\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-downgrade-g9g6xf in resource group rg-np-downgrade-g9g6xf-875q59, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ece011306dab74cd22dd19f1f3c89bbdb930a2321b61be59e4a6a90aae6e3eb","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-downgrade-g9g6xf in resource group rg-np-downgrade-g9g6xf-875q59, error: failed waiting for cluster=\"np-downgrade-g9g6xf\" in resourcegroup=\"rg-np-downgrade-g9g6xf-875q59\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-downgrade-g9g6xf in resource group rg-np-downgrade-g9g6xf-875q59, error: context deadline exceeded"}},{"row_id":"bc191fb4693b7136bd91f9ca2e4318180d1179db6fef16cffbb56d8e519fa8e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2075795979096821760","occurred_at":"2026-07-11T04:14:57Z","failed_at":"e2e","signature_id":"0bbfece7a73d1d79ec9fa7b246d1331b2fa1e8784b0f7f42261ef3ef5d893edf","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc00199a1f8\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-zstream-upgrade-4-22-xddbdf\", caused by: failed waiting for hcpCluster=\"cluster-zstream-4-22-qz9tbp\" in resourcegroup=\"rg-zstream-upgrade-4-22-xddbdf\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-zstream-4-22-qz9tbp in resource group rg-zstream-upgrade-4-22-xddbdf, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-zstream-4-22-qz9tbp\" in resourcegroup=\"rg-zstream-upgrade-4-22-xddbdf\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-zstream-4-22-qz9tbp in resource group rg-zstream-upgrade-4-22-xddbdf, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc00199a1f8\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-zstream-upgrade-4-22-xddbdf\", caused by: failed waiting for hcpCluster=\"cluster-zstream-4-22-qz9tbp\" in resourcegroup=\"rg-zstream-upgrade-4-22-xddbdf\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-zstream-4-22-qz9tbp in resource group rg-zstream-upgrade-4-22-xddbdf, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-zstream-4-22-qz9tbp\" in resourcegroup=\"rg-zstream-upgrade-4-22-xddbdf\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-zstream-4-22-qz9tbp in resource group rg-zstream-upgrade-4-22-xddbdf, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f90cae3228fe72b2427515e7c51834eebdc3e73c0b1aa6bca2abedd53a2e7578","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"ephemeral-osdisk-wdqlbm\", caused by: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-wdqlbm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-wdqlbm, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-wdqlbm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-wdqlbm, error: context deadline exceeded"}},{"row_id":"c01e3bd94efc53ca38469ba9607e59253694efd64a7f9071b081fa1514f5ddbc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2075795979096821760","occurred_at":"2026-07-11T04:14:57Z","failed_at":"e2e","signature_id":"a52cdb4c37ecd3e0bb01e423d1bb47f929c074c9ffd32946a6f3fa7d79c29bb1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-7rhzs8\" for cluster \"cluster-candidate-4-21-7rhzs8\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00278eba0\u003e: \n    failed to create NodePool np-7rhzs8, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-7rhzs8 in resource group rg-candidate-4-21-c5mrcr, error: failed waiting for nodepool=\"np-7rhzs8\" for cluster \"cluster-candidate-4-21-7rhzs8\" in resourcegroup=\"rg-candidate-4-21-c5mrcr\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:159]: failed to create node pool \"np-7rhzs8\" for cluster \"cluster-candidate-4-21-7rhzs8\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00278eba0\u003e: \n    failed to create NodePool np-7rhzs8, caused by: timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-7rhzs8 in resource group rg-candidate-4-21-c5mrcr, error: failed waiting for nodepool=\"np-7rhzs8\" for cluster \"cluster-candidate-4-21-7rhzs8\" in resourcegroup=\"rg-candidate-4-21-c5mrcr\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a880b7dbbe8686372b606e0212fd3742bc75c3c0a9c545bae9481b243ed0f55b","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-7rhzs8 in resource group rg-candidate-4-21-c5mrcr, error: failed waiting for nodepool=\"np-7rhzs8\" for cluster \"cluster-candidate-4-21-7rhzs8\" in resourcegroup=\"rg-candidate-4-21-c5mrcr\" to finish creating: context deadline exceeded"}},{"row_id":"c0ef2ac4d706b8e4326fd925780e8087de1f0fdf470cff1c0ff71a2826c8742e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2075795979096821760","occurred_at":"2026-07-11T04:14:57Z","failed_at":"e2e","signature_id":"90699d43e46d95701ad6ea828fe60e19503360d6286a852fceccd8a0e1152ae0","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc00125a0f0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"ephemeral-osdisk-wdqlbm\", caused by: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-wdqlbm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-wdqlbm, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-wdqlbm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-wdqlbm, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc00125a0f0\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"ephemeral-osdisk-wdqlbm\", caused by: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-wdqlbm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-wdqlbm, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-wdqlbm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-wdqlbm, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f90cae3228fe72b2427515e7c51834eebdc3e73c0b1aa6bca2abedd53a2e7578","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"ephemeral-osdisk-wdqlbm\", caused by: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-wdqlbm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-wdqlbm, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-wdqlbm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-wdqlbm, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["0ece011306dab74cd22dd19f1f3c89bbdb930a2321b61be59e4a6a90aae6e3eb","a880b7dbbe8686372b606e0212fd3742bc75c3c0a9c545bae9481b243ed0f55b","f90cae3228fe72b2427515e7c51834eebdc3e73c0b1aa6bca2abedd53a2e7578"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2075737788124237824","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-11T00:23:48Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d1c48db31e28bc5ec9b5d95a04ad2e285309d7f855dc0a6a63bdf65a7f775084","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2075737788124237824","occurred_at":"2026-07-11T00:23:48Z","failed_at":"e2e","signature_id":"8ec225040ae000d45a9c881d774dfb2b781dffc5ea704607532b22a5019ccceb","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc000538648\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-pndqms\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-pndqms\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-pndqms, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-pndqms\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-pndqms, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc000538648\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-pndqms\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-pndqms\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-pndqms, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-pndqms\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-pndqms, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f90cae3228fe72b2427515e7c51834eebdc3e73c0b1aa6bca2abedd53a2e7578","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"ephemeral-osdisk-wdqlbm\", caused by: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-wdqlbm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-wdqlbm, error: context deadline exceeded, error: failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-wdqlbm\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-wdqlbm, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f90cae3228fe72b2427515e7c51834eebdc3e73c0b1aa6bca2abedd53a2e7578"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}