{"meta":{"start_date":"2026-04-20","end_date":"2026-04-20","start_at":"2026-04-20T00:00:00Z","end_at":"2026-04-21T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T15:42:59Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":1,"failed_runs":1,"raw_occurrences":4,"matched_occurrences":4,"runs_affected":1},"rows":[{"environment":"prod","failure_pattern_id":"0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","failure_pattern":"tls: failed to verify certificate: x509: certificate is valid for \u003chcp-api-host\u003e, \u003chcp-api-host\u003e, not \u003chcp-api-host\u003e","search_query":"tls: failed to verify certificate: x509: certificate is valid for api.o9k4j6p7a1e1h0s.a1gh.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.o9k4j6p7a1e1h0s.60d9.uksouth.aroapp-hcp.io","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-04-14..2026-04-20 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-23","2026-04-06","2026-04-13"],"prior_runs_affected":5,"prior_last_seen_at":"2026-04-15T09:25:00Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:160]: Unexpected error:\n    \u003c*errors.joinError | 0xc001880d20\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.o9k4j6p7a1e1h0s.60d9.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.o9k4j6p7a1e1h0s.a1gh.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.o9k4j6p7a1e1h0s.60d9.uksouth.aroapp-hcp.io\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.o9k4j6p7a1e1h0s.60d9.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.o9k4j6p7a1e1h0s.a1gh.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.o9k4j6p7a1e1h0s.60d9.uksouth.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:160]: Unexpected error:\n    \u003c*errors.joinError | 0xc001880d20\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.o9k4j6p7a1e1h0s.60d9.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.o9k4j6p7a1e1h0s.a1gh.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.o9k4j6p7a1e1h0s.60d9.uksouth.aroapp-hcp.io\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.o9k4j6p7a1e1h0s.60d9.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.o9k4j6p7a1e1h0s.a1gh.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.o9k4j6p7a1e1h0s.60d9.uksouth.aroapp-hcp.io\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:98]: Unexpected error:\n    \u003c*errors.joinError | 0xc00107ef30\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.u0b6k0h2n5h7u1f.ccn5.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.u0b6k0h2n5h7u1f.vrwb.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.u0b6k0h2n5h7u1f.ccn5.uksouth.aroapp-hcp.io\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.u0b6k0h2n5h7u1f.ccn5.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.u0b6k0h2n5h7u1f.vrwb.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.u0b6k0h2n5h7u1f.ccn5.uksouth.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:98]: Unexpected error:\n    \u003c*errors.joinError | 0xc00107ef30\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.u0b6k0h2n5h7u1f.ccn5.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.u0b6k0h2n5h7u1f.vrwb.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.u0b6k0h2n5h7u1f.ccn5.uksouth.aroapp-hcp.io\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.u0b6k0h2n5h7u1f.ccn5.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.u0b6k0h2n5h7u1f.vrwb.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.u0b6k0h2n5h7u1f.ccn5.uksouth.aroapp-hcp.io\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Unexpected error:\n    \u003c*errors.joinError | 0xc001286438\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.r8w7u2u4m1a6h8y.dxvx.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.r8w7u2u4m1a6h8y.f07n.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.r8w7u2u4m1a6h8y.dxvx.uksouth.aroapp-hcp.io\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.r8w7u2u4m1a6h8y.dxvx.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.r8w7u2u4m1a6h8y.f07n.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.r8w7u2u4m1a6h8y.dxvx.uksouth.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Unexpected error:\n    \u003c*errors.joinError | 0xc001286438\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.r8w7u2u4m1a6h8y.dxvx.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.r8w7u2u4m1a6h8y.f07n.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.r8w7u2u4m1a6h8y.dxvx.uksouth.aroapp-hcp.io\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.r8w7u2u4m1a6h8y.dxvx.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.r8w7u2u4m1a6h8y.f07n.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.r8w7u2u4m1a6h8y.dxvx.uksouth.aroapp-hcp.io\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046046207125491712","occurred_at":"2026-04-20T02:00:04Z","signature_id":"1005de253f4229d1b1e34e9d833e0b18059f7431a8b94dc7c31d84a49a268ed5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046046207125491712","occurred_at":"2026-04-20T02:00:04Z","signature_id":"983604bad68098fb0a45fb07045171bcd0da177b95925c426788e8c8ecded3bc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046046207125491712","occurred_at":"2026-04-20T02:00:04Z","signature_id":"da3f7f58d6c27dfd98f711bd046985429377570443c58fba1b1ed57bb45506a4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":100,"anchor_occurrences":1,"after_last_push_count":61,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-04-14..2026-04-20 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-30","2026-04-06"],"prior_runs_affected":34,"prior_last_seen_at":"2026-04-12T07:50:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:35]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:35]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2046046207125491712","occurred_at":"2026-04-20T02:00:04Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}