{"meta":{"date":"2026-07-14","timezone":"UTC","generated_at":"2026-08-27T18:55:12Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":5,"failed_runs":4,"runs_with_occurrences":4,"runs_with_failure_pattern_matches":3,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":1,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2077175575545384960","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-14T23:37:01Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"4d4a4497e4b38e7e68805f96caffdefbe436579a4fc9b15255a59c72aa07b6ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2077175575545384960","occurred_at":"2026-07-14T23:37:01Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2077122454056079360","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-14T20:05:58Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a0231bdc39d54e7b6371f501122d4535a24dfb70c8d5ec5bedd26b877df81b09","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2077122454056079360","occurred_at":"2026-07-14T20:05:58Z","failed_at":"e2e","signature_id":"a960a27a863e913a05fdef48fac588c52bed23ab9a71a1a283c13c96638e9bdc","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:194]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0014903c0\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-bxb9f.apps.aro.cilium-cluster.h044.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:194]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0014903c0\u003e: \n    VerifySimpleWebApp failed: route was never reachable: Get \"https://agnhost-e2e-serving-app-bxb9f.apps.aro.cilium-cluster.h044.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98e32667c3e390e96a18d1f5c467888c22b014b872e3aaaa2bfc4f15efac6c45","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["98e32667c3e390e96a18d1f5c467888c22b014b872e3aaaa2bfc4f15efac6c45"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2077031637907935232","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-07-14T14:05:06Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2076956358430691328","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-14T09:05:58Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"02804f37f92e150b4135f5eacae1f64c97485b5cdbcbdc1165a2b4fcdd1a7921","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2076956358430691328","occurred_at":"2026-07-14T09:05:58Z","failed_at":"e2e","signature_id":"cc7509446673b61828a7c8b7f11d470845176edb557b2f1178e53bf8c9816fd7","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014583a8\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"arm64-vm-cluster-t95wmj\", caused by: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-t95wmj\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-t95wmj, error: context deadline exceeded, error: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-t95wmj\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-t95wmj, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:283]: Unexpected error:\n    \u003c*errors.joinError | 0xc0014583a8\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"arm64-vm-cluster-t95wmj\", caused by: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-t95wmj\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-t95wmj, error: context deadline exceeded, error: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-t95wmj\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-t95wmj, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f90cae3228fe72b2427515e7c51834eebdc3e73c0b1aa6bca2abedd53a2e7578","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"arm64-vm-cluster-t95wmj\", caused by: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-t95wmj\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-t95wmj, error: context deadline exceeded, error: failed waiting for hcpCluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-t95wmj\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-t95wmj, error: context deadline exceeded"}},{"row_id":"2199cb9501a031fa812b92938367bac561f4621525078cc0955f94aa58fea75f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2076956358430691328","occurred_at":"2026-07-14T09:05:58Z","failed_at":"e2e","signature_id":"03ce609557f1471eda8591b768fad24ef6a5fae8f8384a0979aeb606d6e1ac0b","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:131]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00172dc08\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.28) failed: no version in clusterversion/version status.history is greater than initial \"4.20.28\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:131]: Timed out after 2700.004s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00172dc08\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.20.28) failed: no version in clusterversion/version status.history is greater than initial \"4.20.28\"\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"18d5af65735558751d0c56d3c20458f218b143c5bf541874b01f142b2c1ce20c","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4"}},{"row_id":"b4aab9d5e2e7ecb916ed7a125fcc4258c587c9e9d6c6c234b747d27caf5eb6d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2076956358430691328","occurred_at":"2026-07-14T09:05:58Z","failed_at":"e2e","signature_id":"2b56027b295af65d7b607442c212d44167ad35f7328e15e99bcb50811727be3f","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:145]: failed to delete customer resource group \"cx-rg-cluster-g6j8kb\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00167a450\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-g6j8kb, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:145]: failed to delete customer resource group \"cx-rg-cluster-g6j8kb\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc00167a450\u003e: \n    failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-g6j8kb, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8a28d48cc4970d0f2235caa51b21154be46f90f637aecb0006fe72ad7a4ea5","failure_pattern":"DeleteResourceGroup timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to delete resource group, caused by: timeout '60.000000' minutes exceeded during DeleteResourceGroup for resource group cx-rg-cluster-g6j8kb, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["18d5af65735558751d0c56d3c20458f218b143c5bf541874b01f142b2c1ce20c","6d8a28d48cc4970d0f2235caa51b21154be46f90f637aecb0006fe72ad7a4ea5","f90cae3228fe72b2427515e7c51834eebdc3e73c0b1aa6bca2abedd53a2e7578"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2076886089682915328","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-07-14T04:26:45Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"03aa91d4262ba6820580dd2361be96477648011e1c1f2417799b51cf5436276e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2076886089682915328","occurred_at":"2026-07-14T04:26:45Z","failed_at":"e2e","signature_id":"2d623f9849a6044f6e1a1a8d035350fd9d6a63e5d264a58b657f566695e5b50b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-thmlxl\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007e18c0\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-ftkg2.apps.aro.c7k1v4u4a1s2g0i.s1vp.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:172]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-thmlxl\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007e18c0\u003e: \n    route was never reachable: Get \"https://agnhost-e2e-serving-app-ftkg2.apps.aro.c7k1v4u4a1s2g0i.s1vp.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98e32667c3e390e96a18d1f5c467888c22b014b872e3aaaa2bfc4f15efac6c45","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}},{"row_id":"587693b9b02163768ea228f10a6d4c8875f3c62259c9bffb184422a7e85788c7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2076886089682915328","occurred_at":"2026-07-14T04:26:45Z","failed_at":"e2e","signature_id":"209c35f8d9e475597a5aac8f8ae2c180247ed03ac10d97dc841e87b7cd6fc526","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:289]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000978b40\u003e: \n    failed to create HCP cluster node-limit-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-vmtz78, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-vmtz78\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-vmtz78, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:289]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000978b40\u003e: \n    failed to create HCP cluster node-limit-cluster, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-vmtz78, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-vmtz78\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-vmtz78, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ece011306dab74cd22dd19f1f3c89bbdb930a2321b61be59e4a6a90aae6e3eb","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-vmtz78, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-vmtz78\" to finish creating, caused by: timeout '20.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-vmtz78, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["0ece011306dab74cd22dd19f1f3c89bbdb930a2321b61be59e4a6a90aae6e3eb","98e32667c3e390e96a18d1f5c467888c22b014b872e3aaaa2bfc4f15efac6c45"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}