{"meta":{"date":"2026-08-18","timezone":"UTC","generated_at":"2026-08-25T22:08:43Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":6,"failed_runs":4,"runs_with_occurrences":4,"runs_with_failure_pattern_matches":4,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":2,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2089828094020947968","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-18T21:33:39Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"7c4d2535c92d408f077f1cbe7f05525c5e4559d4a2a4dcb7a05752f695d1c1b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2089828094020947968","occurred_at":"2026-08-18T21:33:39Z","failed_at":"e2e","signature_id":"67c67f1508e8b4c5e106485bef9ecc40e2ca93f5f9bd5e6aa4ae684374b5dd8d","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:196]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0018ea198\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-jf5mq.apps.aro.cilium-cluster.exnb.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:196]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0018ea198\u003e: \n    VerifySimpleWebApp failed: [strict TLS verification] route was never reachable: Get \"https://agnhost-e2e-sample-app-jf5mq.apps.aro.cilium-cluster.exnb.uksouth.aroapp-hcp.azure-test.net\": tls: failed to verify certificate: x509: certificate signed by unknown authority\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98e32667c3e390e96a18d1f5c467888c22b014b872e3aaaa2bfc4f15efac6c45","failure_pattern":"tls: failed to verify certificate: x509: certificate signed by unknown authority","search_query":"tls: failed to verify certificate: x509: certificate signed by unknown authority"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["98e32667c3e390e96a18d1f5c467888c22b014b872e3aaaa2bfc4f15efac6c45"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2089783838367551488","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-18T18:37:47Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"be4b8a7105c6e830194db15399e56dc9de2cea3cfa63f9b6b237e82ee8956d96","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2089783838367551488","occurred_at":"2026-08-18T18:37:47Z","failed_at":"e2e","signature_id":"910bbdbc57d54c46c77bac5f7a773bb7a5240de5ea49c4347a70f8ea721f1275","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00098d800\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-txwg4qgs4hsw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/002755ed-c845-4398-a37b-8daa3b7cdc45\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/002755ed-c845-4398-a37b-8daa3b7cdc45\",\n      \"name\": \"002755ed-c845-4398-a37b-8daa3b7cdc45\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T18:50:47.72810349Z\",\n      \"endTime\": \"2026-08-18T19:09:54.253810369Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00098d800\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-txwg4qgs4hsw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/002755ed-c845-4398-a37b-8daa3b7cdc45\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/002755ed-c845-4398-a37b-8daa3b7cdc45\",\n      \"name\": \"002755ed-c845-4398-a37b-8daa3b7cdc45\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T18:50:47.72810349Z\",\n      \"endTime\": \"2026-08-18T19:09:54.253810369Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e0eca219c3110341e80c5a3f64ae984723abc1e1208ac11199b51c76f20de3e7","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e0eca219c3110341e80c5a3f64ae984723abc1e1208ac11199b51c76f20de3e7"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2089731977035911168","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-18T15:11:43Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1efacbce65429950d6e65c9e51f4db5d790946bc9650f739f5b62e8c6a4d7a06","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2089731977035911168","occurred_at":"2026-08-18T15:11:43Z","failed_at":"e2e","signature_id":"20cd25cfb84069bf0dbeae90ccfea853127cb3d7c1e8d8f537a39512ce31cc19","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.000s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:236]: Timed out after 600.000s.\nexpect ingress certificate to be signed by a trusted Azure CA\nExpected success, but got an error:\n    \u003cx509.UnknownAuthorityError\u003e: \n    x509: certificate signed by unknown authority\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"77fb076d32302f92d929786f345bdad3139a0df750404c0633de671aa620060b","failure_pattern":"x509: certificate signed by unknown authority","search_query":"x509: certificate signed by unknown authority"}},{"row_id":"a41e336c7de351b73a991a07d28753f543015e72375554180a8a52ae1b7afa80","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2089731977035911168","occurred_at":"2026-08-18T15:11:43Z","failed_at":"e2e","signature_id":"3e23cbc74631d15c85d630c1ca30b110fee629980794188ea0bfc4db05b38130","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00101a6a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-rkhdl8xqhdmz\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0d86f430-60d6-4c0f-b439-c3ac3403f901\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0d86f430-60d6-4c0f-b439-c3ac3403f901\",\n      \"name\": \"0d86f430-60d6-4c0f-b439-c3ac3403f901\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T16:03:01.445081159Z\",\n      \"endTime\": \"2026-08-18T16:27:09.852869865Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s92v1tvs13qsc1h7oqpk49j9bta62qs still exists (deletion dispatched at 2026-08-18T16:03:02Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:293]: Unexpected error:\n    \u003c*errors.joinError | 0xc00101a6a8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-rkhdl8xqhdmz\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0d86f430-60d6-4c0f-b439-c3ac3403f901\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0d86f430-60d6-4c0f-b439-c3ac3403f901\",\n      \"name\": \"0d86f430-60d6-4c0f-b439-c3ac3403f901\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T16:03:01.445081159Z\",\n      \"endTime\": \"2026-08-18T16:27:09.852869865Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2s92v1tvs13qsc1h7oqpk49j9bta62qs still exists (deletion dispatched at 2026-08-18T16:03:02Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ef4dd01f86a43829d70263dd0a02bc0d1329f80083910a288015623f9842f040","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["77fb076d32302f92d929786f345bdad3139a0df750404c0633de671aa620060b","ef4dd01f86a43829d70263dd0a02bc0d1329f80083910a288015623f9842f040"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2089682863220854784","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-08-18T11:56:33Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"959bd5a98d48978f66ccfa922ad75a1506a13393f1792517f674e390ac97683a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2089682863220854784","occurred_at":"2026-08-18T11:56:33Z","failed_at":"e2e","signature_id":"00cbbf2460a0f78e2867edebb759a4cbf8777cb1f20b47ec15fe40c9b511cc87","test_name":"HCP Nodepools GPU instances creates and deletes a GPU nodepool in a single cluster","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:131]: failed to create GPU nodepool gpu-np-1 with VM size Standard_NC4as_T4_v3\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a9e9c0\u003e: \n    failed to create NodePool gpu-np-1: failed waiting for nodepool=\"gpu-np-1\" for cluster \"cluster-gpu-np\" in resourcegroup=\"rg-gpu-nodepool-6rqw64hz9r29\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df04d9fe-d188-4567-a852-79007382c2f0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df04d9fe-d188-4567-a852-79007382c2f0\",\n      \"name\": \"df04d9fe-d188-4567-a852-79007382c2f0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T13:10:13.902866674Z\",\n      \"endTime\": \"2026-08-18T13:29:15.631640019Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/gpu_nodepools_create_delete.go:131]: failed to create GPU nodepool gpu-np-1 with VM size Standard_NC4as_T4_v3\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a9e9c0\u003e: \n    failed to create NodePool gpu-np-1: failed waiting for nodepool=\"gpu-np-1\" for cluster \"cluster-gpu-np\" in resourcegroup=\"rg-gpu-nodepool-6rqw64hz9r29\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df04d9fe-d188-4567-a852-79007382c2f0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/df04d9fe-d188-4567-a852-79007382c2f0\",\n      \"name\": \"df04d9fe-d188-4567-a852-79007382c2f0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-08-18T13:10:13.902866674Z\",\n      \"endTime\": \"2026-08-18T13:29:15.631640019Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] \\u003cno_message\\u003e\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e0eca219c3110341e80c5a3f64ae984723abc1e1208ac11199b51c76f20de3e7","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] \u003cno_message\u003e; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["e0eca219c3110341e80c5a3f64ae984723abc1e1208ac11199b51c76f20de3e7"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2089611515891027968","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-18T07:13:03Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2089557654589411328","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-08-18T03:39:01Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}