{"meta":{"date":"2026-04-15","timezone":"UTC","generated_at":"2026-08-30T08:19:56Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":7,"failed_runs":4,"runs_with_occurrences":4,"runs_with_failure_pattern_matches":4,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":3,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2044457984683151360","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-15T16:46:01Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2044421226465595392","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-15T14:17:32Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"965fa300f1d10859a8b9ea743f053b43d300ea6782bd2afb7bbee716377b6f4b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2044421226465595392","occurred_at":"2026-04-15T14:17:32Z","failed_at":"e2e","signature_id":"6fd351445b5dec79e0d4ecb2bafc00b005459ea28c5b6f83e24eab135237b8bf","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:262]: Expected success, but got an error:\n    \u003c*errors.errorString | 0xc000ddc1c0\u003e: \n    expected 4 nodes, found 5\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:262]: Expected success, but got an error:\n    \u003c*errors.errorString | 0xc000ddc1c0\u003e: \n    expected 4 nodes, found 5\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6f73b8eb6e6efeec5349b3b80b47a2832fd101f5102faf6f866507896f1bc8fe","failure_pattern":"expected 4 nodes, found 5","search_query":"expected 4 nodes, found 5"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6f73b8eb6e6efeec5349b3b80b47a2832fd101f5102faf6f866507896f1bc8fe"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2044419001106305024","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-15T14:02:57Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"082df33ec902097b655f2bf26ccb7aa26caba9c8d7d9a229bf041b1da536400a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2044419001106305024","occurred_at":"2026-04-15T14:02:57Z","failed_at":"e2e","signature_id":"ae67d67f27d611da0285b354efb2aac747a6dc50ddd4740c3b99652397880c6c","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:186]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00132a1a0\u003e: \n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"e2e-cilium-7jmwwn\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:186]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00132a1a0\u003e: \n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"e2e-cilium-7jmwwn\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"79d555cc6e5bd17f4b96af0c9e2da6a571d8241aacf20f84304c98773ef2682b","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"e2e-cilium-7jmwwn\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["79d555cc6e5bd17f4b96af0c9e2da6a571d8241aacf20f84304c98773ef2682b"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2044346239444586496","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-15T09:25:00Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"b5d66b48ed18364fc5fc4a03e20206338b3485371084d1285f2359d67e9254f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2044346239444586496","occurred_at":"2026-04-15T09:25:00Z","failed_at":"e2e","signature_id":"ae5b3e3e237fb2522c7895a90c73cc83a047d058b702478e3795ea8f25ea6058","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:186]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000118040\u003e: \n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"e2e-cilium-hcd2ng\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:186]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000118040\u003e: \n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"e2e-cilium-hcd2ng\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"79d555cc6e5bd17f4b96af0c9e2da6a571d8241aacf20f84304c98773ef2682b","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"e2e-cilium-7jmwwn\" to finish creating: context deadline exceeded"}},{"row_id":"e732a2f772740228d3cc4d74a54f4bbfc899807d60726ab393327910973cddd3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2044346239444586496","occurred_at":"2026-04-15T09:25:00Z","failed_at":"e2e","signature_id":"6d0d730220c6e446098d300c420715cba29c5c8e6b7235ffcf0e645bcd39d70a","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:161]: Unexpected error:\n    \u003c*url.Error | 0xc00161c840\u003e: \n    Get \"https://api.a5t3f6u4j8a4a5h.o0jt.eastus2.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": tls: failed to verify certificate: x509: certificate is valid for api.a5t3f6u4j8a4a5h.4ufg.eastus2.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.a5t3f6u4j8a4a5h.o0jt.eastus2.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:161]: Unexpected error:\n    \u003c*url.Error | 0xc00161c840\u003e: \n    Get \"https://api.a5t3f6u4j8a4a5h.o0jt.eastus2.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": tls: failed to verify certificate: x509: certificate is valid for api.a5t3f6u4j8a4a5h.4ufg.eastus2.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.a5t3f6u4j8a4a5h.o0jt.eastus2.aroapp-hcp.io\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","failure_pattern":"tls: failed to verify certificate: x509: certificate is valid for \u003chcp-api-host\u003e, \u003chcp-api-host\u003e, not \u003chcp-api-host\u003e","search_query":"tls: failed to verify certificate: x509: certificate is valid for api.a5t3f6u4j8a4a5h.4ufg.eastus2.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.a5t3f6u4j8a4a5h.o0jt.eastus2.aroapp-hcp.io"}},{"row_id":"f0bb069053302f684842e4c8ad91f04e755d4864f4bc068ea26530b9c200f5a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2044346239444586496","occurred_at":"2026-04-15T09:25:00Z","failed_at":"e2e","signature_id":"585d6a3a3e15dc4474a3440e8dd3a198ff732f03a848fed3dda901dabb1310bb","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:262]: Expected success, but got an error:\n    \u003c*errors.errorString | 0xc000ff0ef0\u003e: \n    expected 4 nodes, found 5\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:262]: Expected success, but got an error:\n    \u003c*errors.errorString | 0xc000ff0ef0\u003e: \n    expected 4 nodes, found 5\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6f73b8eb6e6efeec5349b3b80b47a2832fd101f5102faf6f866507896f1bc8fe","failure_pattern":"expected 4 nodes, found 5","search_query":"expected 4 nodes, found 5"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","6f73b8eb6e6efeec5349b3b80b47a2832fd101f5102faf6f866507896f1bc8fe","79d555cc6e5bd17f4b96af0c9e2da6a571d8241aacf20f84304c98773ef2682b"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2044309051059212288","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-15T06:57:13Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2044295773960540160","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-15T06:04:28Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"754d6a4d358a74e43e46449ba409b3962244b1b60e9404279e284b80bf500978","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2044295773960540160","occurred_at":"2026-04-15T06:04:28Z","failed_at":"e2e","signature_id":"f1d5af713ee9b2e1f42d1df9d26db81585c741c316c47e5fa3466f43d7d87a23","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000cb0660\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-dwmdj6, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-dwmdj6\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000cb0660\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-dwmdj6, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-dwmdj6\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-dwmdj6, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-dwmdj6\" to finish creating: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2044234261703168000","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-04-15T02:00:02Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}