{"meta":{"date":"2026-04-23","timezone":"UTC","generated_at":"2026-08-30T06:30:56Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":4,"failed_runs":4,"runs_with_occurrences":4,"runs_with_failure_pattern_matches":4,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047452573052440576","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-23T23:08:27Z"},"failed_at":["e2e"],"failed_test_count":7,"occurrences":[{"row_id":"0670cc3ef7a7b06469a908cdbb7c6ff0a2934a3504488e10ff9ecd3f166c2191","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047452573052440576","occurred_at":"2026-04-23T23:08:27Z","failed_at":"e2e","signature_id":"c0883f339955db3f2f7834e4ee94e66a2fede7e2aaca213690a34560424401af","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:103]: Unexpected error:\n    \u003c*errors.joinError | 0xc001984888\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.q5l8i2i0z6e7p8t.8azc.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.q5l8i2i0z6e7p8t.lohc.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.q5l8i2i0z6e7p8t.8azc.uksouth.aroapp-hcp.io\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.q5l8i2i0z6e7p8t.8azc.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.q5l8i2i0z6e7p8t.lohc.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.q5l8i2i0z6e7p8t.8azc.uksouth.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:103]: Unexpected error:\n    \u003c*errors.joinError | 0xc001984888\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.q5l8i2i0z6e7p8t.8azc.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.q5l8i2i0z6e7p8t.lohc.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.q5l8i2i0z6e7p8t.8azc.uksouth.aroapp-hcp.io\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.q5l8i2i0z6e7p8t.8azc.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.q5l8i2i0z6e7p8t.lohc.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.q5l8i2i0z6e7p8t.8azc.uksouth.aroapp-hcp.io\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","failure_pattern":"tls: failed to verify certificate: x509: certificate is valid for \u003chcp-api-host\u003e, \u003chcp-api-host\u003e, not \u003chcp-api-host\u003e","search_query":"tls: failed to verify certificate: x509: certificate is valid for api.a3p3i1v4e6u2c5w.e9wf.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.a3p3i1v4e6u2c5w.v3e1.uksouth.aroapp-hcp.io"}},{"row_id":"2790d971ce3bede96b3ceb29f5e80de3a74dcb9d363ba484c34895fd0bca461a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047452573052440576","occurred_at":"2026-04-23T23:08:27Z","failed_at":"e2e","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","test_name":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:89]: Expected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:89]: Expected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f7692883b7d8edf2d0661368bcbf0596b25bb01a118dcf71cc62820145611635","failure_pattern":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set: assertion failed: expected values to equal","search_query":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set: assertion failed: expected values to equal"}},{"row_id":"4b47e83a5fb12fc68a31169102ac01e2df115425721f570e802cb0b1a958f4c5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047452573052440576","occurred_at":"2026-04-23T23:08:27Z","failed_at":"e2e","signature_id":"78e2630b9333b13334c561e7f2f476be7dfbb53e9888cce591f72bdb67b90d20","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:160]: Unexpected error:\n    \u003c*errors.joinError | 0xc00104ab10\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.y8r9m4v2u9w1e2o.5d45.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.y8r9m4v2u9w1e2o.lmqw.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.y8r9m4v2u9w1e2o.5d45.uksouth.aroapp-hcp.io\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.y8r9m4v2u9w1e2o.5d45.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.y8r9m4v2u9w1e2o.lmqw.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.y8r9m4v2u9w1e2o.5d45.uksouth.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:160]: Unexpected error:\n    \u003c*errors.joinError | 0xc00104ab10\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.y8r9m4v2u9w1e2o.5d45.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.y8r9m4v2u9w1e2o.lmqw.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.y8r9m4v2u9w1e2o.5d45.uksouth.aroapp-hcp.io\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.y8r9m4v2u9w1e2o.5d45.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.y8r9m4v2u9w1e2o.lmqw.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.y8r9m4v2u9w1e2o.5d45.uksouth.aroapp-hcp.io\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","failure_pattern":"tls: failed to verify certificate: x509: certificate is valid for \u003chcp-api-host\u003e, \u003chcp-api-host\u003e, not \u003chcp-api-host\u003e","search_query":"tls: failed to verify certificate: x509: certificate is valid for api.a3p3i1v4e6u2c5w.e9wf.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.a3p3i1v4e6u2c5w.v3e1.uksouth.aroapp-hcp.io"}},{"row_id":"57b5170f105ed86de33688ac62505a8b2e20a5620f3c4837c30b6914f7a4be8c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047452573052440576","occurred_at":"2026-04-23T23:08:27Z","failed_at":"e2e","signature_id":"2364c917ae0cf12ed69fbb25e4c0413548f8ec6407144980cf6b33fbdae36ac4","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:161]: Unexpected error:\n    \u003c*url.Error | 0xc00174e600\u003e: \n    Get \"https://api.t9i0s9f7m4g0f5e.0122.uksouth.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": tls: failed to verify certificate: x509: certificate is valid for api.t9i0s9f7m4g0f5e.rzw9.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.t9i0s9f7m4g0f5e.0122.uksouth.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:161]: Unexpected error:\n    \u003c*url.Error | 0xc00174e600\u003e: \n    Get \"https://api.t9i0s9f7m4g0f5e.0122.uksouth.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": tls: failed to verify certificate: x509: certificate is valid for api.t9i0s9f7m4g0f5e.rzw9.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.t9i0s9f7m4g0f5e.0122.uksouth.aroapp-hcp.io\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","failure_pattern":"tls: failed to verify certificate: x509: certificate is valid for \u003chcp-api-host\u003e, \u003chcp-api-host\u003e, not \u003chcp-api-host\u003e","search_query":"tls: failed to verify certificate: x509: certificate is valid for api.a3p3i1v4e6u2c5w.e9wf.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.a3p3i1v4e6u2c5w.v3e1.uksouth.aroapp-hcp.io"}},{"row_id":"74884b44f2af0edddbcc54e9c6e45f2fec480882a5e7f15f6689d0be0f2d4ed6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047452573052440576","occurred_at":"2026-04-23T23:08:27Z","failed_at":"e2e","signature_id":"7e06bea11cbb64d5a45c82a4c15051b8638201463ad777ef0ac3d9f5a30d3672","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:127]: Unexpected error:\n    \u003c*errors.joinError | 0xc000ef6960\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.a3p3i1v4e6u2c5w.v3e1.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.a3p3i1v4e6u2c5w.e9wf.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.a3p3i1v4e6u2c5w.v3e1.uksouth.aroapp-hcp.io\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.a3p3i1v4e6u2c5w.v3e1.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.a3p3i1v4e6u2c5w.e9wf.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.a3p3i1v4e6u2c5w.v3e1.uksouth.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:127]: Unexpected error:\n    \u003c*errors.joinError | 0xc000ef6960\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.a3p3i1v4e6u2c5w.v3e1.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.a3p3i1v4e6u2c5w.e9wf.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.a3p3i1v4e6u2c5w.v3e1.uksouth.aroapp-hcp.io\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.a3p3i1v4e6u2c5w.v3e1.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.a3p3i1v4e6u2c5w.e9wf.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.a3p3i1v4e6u2c5w.v3e1.uksouth.aroapp-hcp.io\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","failure_pattern":"tls: failed to verify certificate: x509: certificate is valid for \u003chcp-api-host\u003e, \u003chcp-api-host\u003e, not \u003chcp-api-host\u003e","search_query":"tls: failed to verify certificate: x509: certificate is valid for api.a3p3i1v4e6u2c5w.e9wf.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.a3p3i1v4e6u2c5w.v3e1.uksouth.aroapp-hcp.io"}},{"row_id":"bd34cf970650fb535d0abf9b555034bc110198e972f833c817c311dc0cdd3b16","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047452573052440576","occurred_at":"2026-04-23T23:08:27Z","failed_at":"e2e","signature_id":"8128997d224e82a57c3c7748dd9d74ef03e04eea3cb9a5861a74bf5ab4902946","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:108]: Unexpected error:\n    \u003c*errors.joinError | 0xc000d7c2a0\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.l1h0l2r4w5p9y7a.jwmn.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.l1h0l2r4w5p9y7a.hr9b.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.l1h0l2r4w5p9y7a.jwmn.uksouth.aroapp-hcp.io\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.l1h0l2r4w5p9y7a.jwmn.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.l1h0l2r4w5p9y7a.hr9b.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.l1h0l2r4w5p9y7a.jwmn.uksouth.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:108]: Unexpected error:\n    \u003c*errors.joinError | 0xc000d7c2a0\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.l1h0l2r4w5p9y7a.jwmn.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.l1h0l2r4w5p9y7a.hr9b.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.l1h0l2r4w5p9y7a.jwmn.uksouth.aroapp-hcp.io\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.l1h0l2r4w5p9y7a.jwmn.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.l1h0l2r4w5p9y7a.hr9b.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.l1h0l2r4w5p9y7a.jwmn.uksouth.aroapp-hcp.io\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","failure_pattern":"tls: failed to verify certificate: x509: certificate is valid for \u003chcp-api-host\u003e, \u003chcp-api-host\u003e, not \u003chcp-api-host\u003e","search_query":"tls: failed to verify certificate: x509: certificate is valid for api.a3p3i1v4e6u2c5w.e9wf.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.a3p3i1v4e6u2c5w.v3e1.uksouth.aroapp-hcp.io"}},{"row_id":"f32bb8dce185214f00e1b2d31aa081ed24e7e2bf568154bf85f004b5b2991089","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047452573052440576","occurred_at":"2026-04-23T23:08:27Z","failed_at":"e2e","signature_id":"7854c7962454d39e05f1f3f3d9816cb4edb55d46ae034464371911e60b844295","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:108]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f427e0\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.t1y2o7l3w2e5r5q.p2mn.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.t1y2o7l3w2e5r5q.viss.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.t1y2o7l3w2e5r5q.p2mn.uksouth.aroapp-hcp.io\n    VerifyImageRegistryDisabled failed: wrong type of error: *url.Error, Get \"https://api.t1y2o7l3w2e5r5q.p2mn.uksouth.aroapp-hcp.io:443/api/v1/namespaces/openshift-image-registry/services/image-registry\": tls: failed to verify certificate: x509: certificate is valid for api.t1y2o7l3w2e5r5q.viss.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.t1y2o7l3w2e5r5q.p2mn.uksouth.aroapp-hcp.io\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.t1y2o7l3w2e5r5q.p2mn.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.t1y2o7l3w2e5r5q.viss.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.t1y2o7l3w2e5r5q.p2mn.uksouth.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:108]: Unexpected error:\n    \u003c*errors.joinError | 0xc000f427e0\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.t1y2o7l3w2e5r5q.p2mn.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.t1y2o7l3w2e5r5q.viss.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.t1y2o7l3w2e5r5q.p2mn.uksouth.aroapp-hcp.io\n    VerifyImageRegistryDisabled failed: wrong type of error: *url.Error, Get \"https://api.t1y2o7l3w2e5r5q.p2mn.uksouth.aroapp-hcp.io:443/api/v1/namespaces/openshift-image-registry/services/image-registry\": tls: failed to verify certificate: x509: certificate is valid for api.t1y2o7l3w2e5r5q.viss.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.t1y2o7l3w2e5r5q.p2mn.uksouth.aroapp-hcp.io\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.t1y2o7l3w2e5r5q.p2mn.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.t1y2o7l3w2e5r5q.viss.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.t1y2o7l3w2e5r5q.p2mn.uksouth.aroapp-hcp.io\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","failure_pattern":"tls: failed to verify certificate: x509: certificate is valid for \u003chcp-api-host\u003e, \u003chcp-api-host\u003e, not \u003chcp-api-host\u003e","search_query":"tls: failed to verify certificate: x509: certificate is valid for api.a3p3i1v4e6u2c5w.e9wf.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.a3p3i1v4e6u2c5w.v3e1.uksouth.aroapp-hcp.io"}}],"failure_pattern_summary":{"signature_count":7,"failure_pattern_ids":["0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","f7692883b7d8edf2d0661368bcbf0596b25bb01a118dcf71cc62820145611635"],"matched_occurrences":7,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047328993031491584","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-23T14:57:24Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"10bb1be281657f9a20a834cdd8867ab877113c4966e3a29f525918d21003080f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047328993031491584","occurred_at":"2026-04-23T14:57:24Z","failed_at":"e2e","signature_id":"c1da7bef0bb2e4746842f7e901cf5beb839a2a61345ef0c4619c637b5f1bd559","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Unexpected error:\n    \u003c*errors.joinError | 0xc000012cd8\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.t9y2f8k0a0s5v9s.ksq8.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.t9y2f8k0a0s5v9s.5zke.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.t9y2f8k0a0s5v9s.ksq8.uksouth.aroapp-hcp.io\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.t9y2f8k0a0s5v9s.ksq8.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.t9y2f8k0a0s5v9s.5zke.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.t9y2f8k0a0s5v9s.ksq8.uksouth.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Unexpected error:\n    \u003c*errors.joinError | 0xc000012cd8\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.t9y2f8k0a0s5v9s.ksq8.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.t9y2f8k0a0s5v9s.5zke.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.t9y2f8k0a0s5v9s.ksq8.uksouth.aroapp-hcp.io\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.t9y2f8k0a0s5v9s.ksq8.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.t9y2f8k0a0s5v9s.5zke.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.t9y2f8k0a0s5v9s.ksq8.uksouth.aroapp-hcp.io\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","failure_pattern":"tls: failed to verify certificate: x509: certificate is valid for \u003chcp-api-host\u003e, \u003chcp-api-host\u003e, not \u003chcp-api-host\u003e","search_query":"tls: failed to verify certificate: x509: certificate is valid for api.a3p3i1v4e6u2c5w.e9wf.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.a3p3i1v4e6u2c5w.v3e1.uksouth.aroapp-hcp.io"}},{"row_id":"1833be1f6946b3f6aa8df17dee22a8ec294d47a23c9dd6d95cb7744fbda70c37","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047328993031491584","occurred_at":"2026-04-23T14:57:24Z","failed_at":"e2e","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","test_name":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:89]: Expected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:89]: Expected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f7692883b7d8edf2d0661368bcbf0596b25bb01a118dcf71cc62820145611635","failure_pattern":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set: assertion failed: expected values to equal","search_query":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set: assertion failed: expected values to equal"}},{"row_id":"298f989f19ea693e6b7bac580fb68fe3b30fcde5cd3a0add5e731c3980c80772","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047328993031491584","occurred_at":"2026-04-23T14:57:24Z","failed_at":"e2e","signature_id":"e8dd1822829579cc99aebe864a873cc56f69bff8d1e602795abe27828cd5b6ef","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:108]: Unexpected error:\n    \u003c*errors.joinError | 0xc001026d98\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.t3j0r5a0h5f9i5x.7i35.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.t3j0r5a0h5f9i5x.wsa2.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.t3j0r5a0h5f9i5x.7i35.uksouth.aroapp-hcp.io\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.t3j0r5a0h5f9i5x.7i35.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.t3j0r5a0h5f9i5x.wsa2.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.t3j0r5a0h5f9i5x.7i35.uksouth.aroapp-hcp.io\n    VerifyImageRegistryDisabled failed: wrong type of error: *url.Error, Get \"https://api.t3j0r5a0h5f9i5x.7i35.uksouth.aroapp-hcp.io:443/api/v1/namespaces/openshift-image-registry/services/image-registry\": tls: failed to verify certificate: x509: certificate is valid for api.t3j0r5a0h5f9i5x.wsa2.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.t3j0r5a0h5f9i5x.7i35.uksouth.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:108]: Unexpected error:\n    \u003c*errors.joinError | 0xc001026d98\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.t3j0r5a0h5f9i5x.7i35.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.t3j0r5a0h5f9i5x.wsa2.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.t3j0r5a0h5f9i5x.7i35.uksouth.aroapp-hcp.io\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.t3j0r5a0h5f9i5x.7i35.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.t3j0r5a0h5f9i5x.wsa2.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.t3j0r5a0h5f9i5x.7i35.uksouth.aroapp-hcp.io\n    VerifyImageRegistryDisabled failed: wrong type of error: *url.Error, Get \"https://api.t3j0r5a0h5f9i5x.7i35.uksouth.aroapp-hcp.io:443/api/v1/namespaces/openshift-image-registry/services/image-registry\": tls: failed to verify certificate: x509: certificate is valid for api.t3j0r5a0h5f9i5x.wsa2.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.t3j0r5a0h5f9i5x.7i35.uksouth.aroapp-hcp.io\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","failure_pattern":"tls: failed to verify certificate: x509: certificate is valid for \u003chcp-api-host\u003e, \u003chcp-api-host\u003e, not \u003chcp-api-host\u003e","search_query":"tls: failed to verify certificate: x509: certificate is valid for api.a3p3i1v4e6u2c5w.e9wf.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.a3p3i1v4e6u2c5w.v3e1.uksouth.aroapp-hcp.io"}},{"row_id":"b83565ebf92ea8373a0264f21ad34345e294e719ebecab0dcb86220b9341e75a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047328993031491584","occurred_at":"2026-04-23T14:57:24Z","failed_at":"e2e","signature_id":"9bef192e78ada1b3ccf602133b1cd3333a57f42a2080d6ca2240cf5ca3bffd8a","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008ec4b0\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-htjcmf, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-htjcmf\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0008ec4b0\u003e: \n    failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-htjcmf, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-htjcmf\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group private-keyvault-htjcmf, error: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-htjcmf\" to finish creating: context deadline exceeded"}},{"row_id":"bf57d071f008f8c941baaf3660190baaeebfbb1f6249ae6fdef1833804b5a96c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047328993031491584","occurred_at":"2026-04-23T14:57:24Z","failed_at":"e2e","signature_id":"18e48067dd9948d3342a0d59d3cc16a366a1d457a85af56e2e788cd9a1e689a9","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:136]: Unexpected error:\n    \u003c*errors.joinError | 0xc001792990\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.x1r8i2z2l1w4b6m.se7s.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.x1r8i2z2l1w4b6m.7c6c.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.x1r8i2z2l1w4b6m.se7s.uksouth.aroapp-hcp.io\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.x1r8i2z2l1w4b6m.se7s.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.x1r8i2z2l1w4b6m.7c6c.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.x1r8i2z2l1w4b6m.se7s.uksouth.aroapp-hcp.io\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:136]: Unexpected error:\n    \u003c*errors.joinError | 0xc001792990\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.x1r8i2z2l1w4b6m.se7s.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.x1r8i2z2l1w4b6m.7c6c.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.x1r8i2z2l1w4b6m.se7s.uksouth.aroapp-hcp.io\n    VerifyBasicAccess failed: failed to list services: Get \"https://api.x1r8i2z2l1w4b6m.se7s.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.x1r8i2z2l1w4b6m.7c6c.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.x1r8i2z2l1w4b6m.se7s.uksouth.aroapp-hcp.io\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","failure_pattern":"tls: failed to verify certificate: x509: certificate is valid for \u003chcp-api-host\u003e, \u003chcp-api-host\u003e, not \u003chcp-api-host\u003e","search_query":"tls: failed to verify certificate: x509: certificate is valid for api.a3p3i1v4e6u2c5w.e9wf.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.a3p3i1v4e6u2c5w.v3e1.uksouth.aroapp-hcp.io"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","f7692883b7d8edf2d0661368bcbf0596b25bb01a118dcf71cc62820145611635"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047231067995246592","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-23T08:28:16Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"27b6a0134b10689c1d887c1a2899eb3e21e9b75b74cba0d9824fecb6de69ae37","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047231067995246592","occurred_at":"2026-04-23T08:28:16Z","failed_at":"e2e","signature_id":"c8cfb15f0633bd15ae0ff18324df4f7cd8aae4df9cd2da599a5448c61107a212","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc000060d20\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-wt4d9m\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc000060d20\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-wt4d9m\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"a558561501317b8bc107720ead7c186571a04877475a317ff377854be7f269ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047231067995246592","occurred_at":"2026-04-23T08:28:16Z","failed_at":"e2e","signature_id":"9300232222f6e824bbddab2e1e764dd788e1b18d31299fc32f6862a0833e5347","test_name":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:89]: Expected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/mise_routing.go:89]: Expected\n    \u003cstring\u003e: ...\nto equal\n    \u003cstring\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f7692883b7d8edf2d0661368bcbf0596b25bb01a118dcf71cc62820145611635","failure_pattern":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set: assertion failed: expected values to equal","search_query":"MISE Routing routes to the correct frontend based on version header MISE v2 when x-ms-mise-version header is set: assertion failed: expected values to equal"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","f7692883b7d8edf2d0661368bcbf0596b25bb01a118dcf71cc62820145611635"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047105181333590016","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-23T00:08:03Z"},"failed_at":["e2e"],"failed_test_count":9,"occurrences":[{"row_id":"232c6fbee309f91439347a12f53ad09aabb5c05dd9270f7778e5285d08dd9611","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047105181333590016","occurred_at":"2026-04-23T00:08:03Z","failed_at":"e2e","signature_id":"8bfd559f19457b829e6cb216b527b9689411e98b3a294af6759a37cad1dcf1c7","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:84]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e26720\u003e: \n    failed to create HCP cluster cilium-cl, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cilium-cl in resource group cni-cilium-wxgvk4, error: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-wxgvk4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cilium-cl in resource group cni-cilium-wxgvk4, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:84]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000e26720\u003e: \n    failed to create HCP cluster cilium-cl, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cilium-cl in resource group cni-cilium-wxgvk4, error: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-wxgvk4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cilium-cl in resource group cni-cilium-wxgvk4, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-pl5xgz, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-pl5xgz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-pl5xgz, error: context deadline exceeded"}},{"row_id":"5581205d1d6e9841de02a49a8e23b5e16175b73ddf9012ba4eb4b4c2fabe9861","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047105181333590016","occurred_at":"2026-04-23T00:08:03Z","failed_at":"e2e","signature_id":"4bd2ef23f572ca3184bd95de7ab7614776b5c577a355de7621203178636245c1","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0017b0570\u003e: \n    failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-fsqjp5, error: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-fsqjp5\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-fsqjp5, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0017b0570\u003e: \n    failed to create HCP cluster patch-tags-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-fsqjp5, error: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-fsqjp5\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-tags-cluster in resource group patch-tags-fsqjp5, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-pl5xgz, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-pl5xgz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-pl5xgz, error: context deadline exceeded"}},{"row_id":"88a9ec70e9cd4aae7d191217aac13119adae6dd3eaf01ea6b070c106d41eda6c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047105181333590016","occurred_at":"2026-04-23T00:08:03Z","failed_at":"e2e","signature_id":"ca3246957b750b911a874eb97e686489d8ea160b690c347586996411de708086","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ffc240\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-pl5xgz, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-pl5xgz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-pl5xgz, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000ffc240\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-pl5xgz, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-pl5xgz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-pl5xgz, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-pl5xgz, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-pl5xgz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-pl5xgz, error: context deadline exceeded"}},{"row_id":"92cf6de72249c6ab735377113118801ae4e6de92839adcce02f8b4088f77769d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047105181333590016","occurred_at":"2026-04-23T00:08:03Z","failed_at":"e2e","signature_id":"3b38a593ad5e9f3ef36d5928f64a9da19e929bfb26b6cc5b8fa32c7782156a79","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001592840\u003e: \n    failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-rpl9lv, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-rpl9lv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-rpl9lv, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc001592840\u003e: \n    failed to create HCP cluster hcp-cluster-np-128, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-rpl9lv, error: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-rpl9lv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster hcp-cluster-np-128 in resource group clusternp128-rpl9lv, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-pl5xgz, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-pl5xgz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-pl5xgz, error: context deadline exceeded"}},{"row_id":"99eba520bc9536bb10b371ef7f943edca2a37f1ad66213f5bf77054b1601e059","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047105181333590016","occurred_at":"2026-04-23T00:08:03Z","failed_at":"e2e","signature_id":"b15c55d32190fb3d32f16f58a52f080055a52cddf394a6cd7e062c8467f788e4","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: Unexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: Unexpected error:\n    \u003ccontext.deadlineExceededError\u003e: \n    context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9626adf85231cae7571909c00084bdfb7c9de3df3208a3d5200ee95baf8fc7e8","failure_pattern":"context deadline exceeded","search_query":"context deadline exceeded"}},{"row_id":"9cc9d69adcdbcebb2c75c02955c9d1706a49a9a0104d72435df222b6d8c63abb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047105181333590016","occurred_at":"2026-04-23T00:08:03Z","failed_at":"e2e","signature_id":"0c4ed9bba8fe112ea7971d2241bf4fe4d2d0df6edae20700d459cff3d63baf26","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.001s.\nCluster should become ready within 45 minutes\nExpected\n    \u003cbool\u003e: ...\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes"}},{"row_id":"b4bad2b30d48849dc21f71a07b8f400cba0620f775452724852e3bdd6e7f2026","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047105181333590016","occurred_at":"2026-04-23T00:08:03Z","failed_at":"e2e","signature_id":"d9a36fe3f7dc5d9825e4ef35af6f2ecf4445891a636467cef49b6c627f4134bf","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010783f0\u003e: \n    failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-q29rcv, error: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-q29rcv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-q29rcv, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0010783f0\u003e: \n    failed to create HCP cluster patch-name-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-q29rcv, error: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-q29rcv\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster patch-name-cluster in resource group patch-name-q29rcv, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-pl5xgz, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-pl5xgz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-pl5xgz, error: context deadline exceeded"}},{"row_id":"e7178c47cb4e954dc48fdc77d155429d7db0fdbe149c52c87f579d604e484b13","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047105181333590016","occurred_at":"2026-04-23T00:08:03Z","failed_at":"e2e","signature_id":"4765fac81880279dbb3c196a6b3dbc41f966dbe9b3d77391790f24ca99363ace","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000fd6d50\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-qsxr69, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-qsxr69\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-qsxr69, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:133]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000fd6d50\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-qsxr69, error: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-qsxr69\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster pullsecret-hcp-cluster in resource group pullsecret-test-qsxr69, error: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-pl5xgz, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-pl5xgz\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-pl5xgz, error: context deadline exceeded"}},{"row_id":"e9710f2b31c17ff3d5c9412188b21fd7ca3ec1e5bd3456ec5c760a0ef1b7ae08","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2047105181333590016","occurred_at":"2026-04-23T00:08:03Z","failed_at":"e2e","signature_id":"0be642cb7b1b860a56a9c13ab1eac783e14b5f53c75ccf658ee52fd8c0807f6e","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc001242648\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-qkvnrj\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: Unexpected error:\n    \u003c*errors.joinError | 0xc001242648\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-qkvnrj\" to finish creating: context deadline exceeded\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":9,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","9626adf85231cae7571909c00084bdfb7c9de3df3208a3d5200ee95baf8fc7e8","afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743"],"matched_occurrences":9,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}