{"meta":{"start_date":"2026-04-15","end_date":"2026-04-15","start_at":"2026-04-15T00:00:00Z","end_at":"2026-04-16T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T18:04:03Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":7,"failed_runs":4,"raw_occurrences":6,"matched_occurrences":6,"runs_affected":4},"rows":[{"environment":"prod","failure_pattern_id":"6f73b8eb6e6efeec5349b3b80b47a2832fd101f5102faf6f866507896f1bc8fe","failure_pattern":"expected 4 nodes, found 5","search_query":"expected 4 nodes, found 5","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":28.571428571428573,"anchor_occurrences":2,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,2,2],"trend_range":"2026-04-09..2026-04-15 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:262]: Expected success, but got an error:\n    \u003c*errors.errorString | 0xc000ddc1c0\u003e: \n    expected 4 nodes, found 5\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:262]: Expected success, but got an error:\n    \u003c*errors.errorString | 0xc000ddc1c0\u003e: \n    expected 4 nodes, found 5\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:262]: Expected success, but got an error:\n    \u003c*errors.errorString | 0xc000ff0ef0\u003e: \n    expected 4 nodes, found 5\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:262]: Expected success, but got an error:\n    \u003c*errors.errorString | 0xc000ff0ef0\u003e: \n    expected 4 nodes, found 5\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2044421226465595392","occurred_at":"2026-04-15T14:17:32Z","signature_id":"6fd351445b5dec79e0d4ecb2bafc00b005459ea28c5b6f83e24eab135237b8bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2044346239444586496","occurred_at":"2026-04-15T09:25:00Z","signature_id":"585d6a3a3e15dc4474a3440e8dd3a198ff732f03a848fed3dda901dabb1310bb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"79d555cc6e5bd17f4b96af0c9e2da6a571d8241aacf20f84304c98773ef2682b","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"e2e-cilium-7jmwwn\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":28.571428571428573,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-04-09..2026-04-15 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-30"],"prior_runs_affected":3,"prior_last_seen_at":"2026-04-05T02:00:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:186]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00132a1a0\u003e: \n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"e2e-cilium-7jmwwn\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:186]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc00132a1a0\u003e: \n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"e2e-cilium-7jmwwn\" to finish creating: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:186]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000118040\u003e: \n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"e2e-cilium-hcd2ng\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:186]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000118040\u003e: \n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"e2e-cilium-hcd2ng\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2044419001106305024","occurred_at":"2026-04-15T14:02:57Z","signature_id":"ae67d67f27d611da0285b354efb2aac747a6dc50ddd4740c3b99652397880c6c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2044346239444586496","occurred_at":"2026-04-15T09:25:00Z","signature_id":"ae5b3e3e237fb2522c7895a90c73cc83a047d058b702478e3795ea8f25ea6058","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-dwmdj6, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-dwmdj6\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":14.285714285714286,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-04-09..2026-04-15 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-16","2026-04-06"],"prior_runs_affected":9,"prior_last_seen_at":"2026-04-12T07:50:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000cb0660\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-dwmdj6, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-dwmdj6\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000cb0660\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-dwmdj6, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-dwmdj6\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2044295773960540160","occurred_at":"2026-04-15T06:04:28Z","signature_id":"f1d5af713ee9b2e1f42d1df9d26db81585c741c316c47e5fa3466f43d7d87a23","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","failure_pattern":"tls: failed to verify certificate: x509: certificate is valid for \u003chcp-api-host\u003e, \u003chcp-api-host\u003e, not \u003chcp-api-host\u003e","search_query":"tls: failed to verify certificate: x509: certificate is valid for api.a5t3f6u4j8a4a5h.4ufg.eastus2.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.a5t3f6u4j8a4a5h.o0jt.eastus2.aroapp-hcp.io","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":14.285714285714286,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[2,0,0,1,0,0,1],"trend_range":"2026-04-09..2026-04-15 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-23","2026-04-06"],"prior_runs_affected":4,"prior_last_seen_at":"2026-04-12T22:01:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:161]: Unexpected error:\n    \u003c*url.Error | 0xc00161c840\u003e: \n    Get \"https://api.a5t3f6u4j8a4a5h.o0jt.eastus2.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": tls: failed to verify certificate: x509: certificate is valid for api.a5t3f6u4j8a4a5h.4ufg.eastus2.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.a5t3f6u4j8a4a5h.o0jt.eastus2.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:161]: Unexpected error:\n    \u003c*url.Error | 0xc00161c840\u003e: \n    Get \"https://api.a5t3f6u4j8a4a5h.o0jt.eastus2.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": tls: failed to verify certificate: x509: certificate is valid for api.a5t3f6u4j8a4a5h.4ufg.eastus2.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.a5t3f6u4j8a4a5h.o0jt.eastus2.aroapp-hcp.io\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2044346239444586496","occurred_at":"2026-04-15T09:25:00Z","signature_id":"6d0d730220c6e446098d300c420715cba29c5c8e6b7235ffcf0e645bcd39d70a","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}