{"meta":{"date":"2026-10-24","timezone":"UTC","generated_at":"2026-10-28T23:56:21Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":2,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2114129822648111104","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-24T23:00:02Z","started_at":"2026-10-24T23:00:02Z","completed_at":"2026-10-25T00:45:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-25T00:59:56.266210936Z","timing_metadata_checked_at":"2026-10-25T00:59:56.266210936Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-25T00:59:56.270271824Z","region_metadata_checked_at":"2026-10-25T00:59:56.270271824Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"711bc8a1a5e971c229c60d74a377b8984483d090e043bf666d51942bf63864be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2114129822648111104","occurred_at":"2026-10-24T23:00:02Z","failed_at":"e2e","signature_id":"ddcef7f1372fb43fa0a1e2ed4da6ae9b117c2451538d5508f48d3045f3f05809","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x2874318ca1e0\u003e: \n    resource group \"rg-nightly-4-21-kv7pzz4ch49v\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-4-21-7x886k\" in resourcegroup=\"rg-nightly-4-21-kv7pzz4ch49v\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35ebc587-6a8d-48f1-a4a4-b752ee43dd71\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35ebc587-6a8d-48f1-a4a4-b752ee43dd71\",\n      \"name\": \"35ebc587-6a8d-48f1-a4a4-b752ee43dd71\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-24T23:51:18.597955269Z\",\n      \"endTime\": \"2026-10-25T00:15:22.050322008Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35ebc587-6a8d-48f1-a4a4-b752ee43dd71\n            correlation ID: e4ebdf9b-312f-4c03-affc-b5c4cbd39c75\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:310]: Unexpected error:\n    \u003c*errors.joinError | 0x2874318ca1e0\u003e: \n    resource group \"rg-nightly-4-21-kv7pzz4ch49v\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-nightly-4-21-7x886k\" in resourcegroup=\"rg-nightly-4-21-kv7pzz4ch49v\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35ebc587-6a8d-48f1-a4a4-b752ee43dd71\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35ebc587-6a8d-48f1-a4a4-b752ee43dd71\",\n      \"name\": \"35ebc587-6a8d-48f1-a4a4-b752ee43dd71\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-24T23:51:18.597955269Z\",\n      \"endTime\": \"2026-10-25T00:15:22.050322008Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 3 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down pod network instances (1), pod networks (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/35ebc587-6a8d-48f1-a4a4-b752ee43dd71\n            correlation ID: e4ebdf9b-312f-4c03-affc-b5c4cbd39c75\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ebbb69cb9ed47c033ebf34b38fd75c121d50dc5d28239fa92e2813d822718c77","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [resourceTeardown] tearing down pod network instances, pod networks, namespaces; [descendantResources] remaining resources: applydesires, serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"833487e83ccaa09d46285e0aa6bdb28746c7ac5a22fed648de49399c9682a705","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2114129822648111104","occurred_at":"2026-10-24T23:00:02Z","failed_at":"e2e","signature_id":"4a296b8184626c9cf0da738c096b9a7efd69214488b2d8d7c86178f33312d094","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:140]: failed to verify HCP cluster \"cluster-nightly-4-22-67pgbb\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0x21cecb2413f8\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x21cecb85e9a0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x21cecb85e980\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x21cecae4e500\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x21cecb471000\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x21cecb470fe0\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x21cecb1806e0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:140]: failed to verify HCP cluster \"cluster-nightly-4-22-67pgbb\" is viable\nUnexpected error:\n    \u003c*errors.joinError | 0x21cecb2413f8\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x21cecb85e9a0\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x21cecb85e980\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x21cecae4e500\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x21cecb471000\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x21cecb470fe0\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x21cecb1806e0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fee9bdd5d83c8738b8bf37e9ed6800aab39015cbc2ff9fc672a130098b9fd967","failure_pattern":"VerifyBasicAccess failed: failed to list services: Unauthorized","search_query":"VerifyBasicAccess failed: failed to list services: Unauthorized"}},{"row_id":"99a067cff56591ab161840073c78a7d1c7cc8be1361bd404d3831c2543070158","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2114129822648111104","occurred_at":"2026-10-24T23:00:02Z","failed_at":"e2e","signature_id":"f4dbd9fc7f76d3d491864efc557c3c22f1a4295a0221ba7f6fc875ded0a8dd16","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:124]: cluster negative-tests-cluster is not viable\nUnexpected error:\n    \u003c*errors.joinError | 0x13bc429c0870\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x13bc42aa9920\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x13bc42aa9900\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x13bc41a5d400\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x13bc425c16c0\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x13bc425c16a0\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x13bc416d8960\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:124]: cluster negative-tests-cluster is not viable\nUnexpected error:\n    \u003c*errors.joinError | 0x13bc429c0870\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x13bc42aa9920\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x13bc42aa9900\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x13bc41a5d400\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x13bc425c16c0\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x13bc425c16a0\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x13bc416d8960\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fee9bdd5d83c8738b8bf37e9ed6800aab39015cbc2ff9fc672a130098b9fd967","failure_pattern":"VerifyBasicAccess failed: failed to list services: Unauthorized","search_query":"VerifyBasicAccess failed: failed to list services: Unauthorized"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["ebbb69cb9ed47c033ebf34b38fd75c121d50dc5d28239fa92e2813d822718c77","fee9bdd5d83c8738b8bf37e9ed6800aab39015cbc2ff9fc672a130098b9fd967"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-24T02:00:02Z","started_at":"2026-10-24T02:00:02Z","completed_at":"2026-10-24T04:14:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-24T04:25:22.231416704Z","timing_metadata_checked_at":"2026-10-24T04:25:22.231416704Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-24T04:25:22.235904691Z","region_metadata_checked_at":"2026-10-24T04:25:22.235904691Z"},"failed_at":["e2e"],"failed_test_count":10,"occurrences":[{"row_id":"0f8ca789108ca619fa4636c64ccb789f6a4423a51fd7066d8b68d0f2084f624e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","failed_at":"e2e","signature_id":"b1dfe79b23dde219591e24160c9becc4dd2e4815fc615b83f459dd71654c29ae","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-23-hvm2zb\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3a2d820f0740\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.009s: getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.009s: getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n        err: \u003c*fmt.wrapError | 0x3a2d820f0700\u003e{\n            msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n            err: \u003c*fmt.wrapError | 0x3a2d820f06c0\u003e{\n                msg: \"client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n                err: \u003c*errors.errorString | 0x3a2d82526860\u003e{\n                    s: \"rate: Wait(n=1) would exceed context deadline\",\n                },\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-23-hvm2zb\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3a2d820f0740\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.009s: getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.009s: getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n        err: \u003c*fmt.wrapError | 0x3a2d820f0700\u003e{\n            msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n            err: \u003c*fmt.wrapError | 0x3a2d820f06c0\u003e{\n                msg: \"client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\",\n                err: \u003c*errors.errorString | 0x3a2d82526860\u003e{\n                    s: \"rate: Wait(n=1) would exceed context deadline\",\n                },\n            },\n        },\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ac84d34217ffed7bf1fb67dcf92fd423dbf5956ae0f518883c8a8dd3cbad89a8","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0"}},{"row_id":"212632ec3d7c3ca143a28f3e73a0f6b6714ce684c5de49395b521991e71f8807","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","failed_at":"e2e","signature_id":"c38d06858d691d9e4c16513cdbf71adc44dfc0e2e5bba3c53da574b0cb0fcaa2","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-20-m5lclb\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x12d085988600\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.007s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.007s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x12d085c685e0\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-20-m5lclb\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x12d085988600\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.007s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.007s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x12d085c685e0\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ac84d34217ffed7bf1fb67dcf92fd423dbf5956ae0f518883c8a8dd3cbad89a8","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0"}},{"row_id":"246cd339725d3e2be9f99da75986ac0877c5d538d8d344e969fa090c8393bd37","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","failed_at":"e2e","signature_id":"f2369be4ef1db102806ff130f76a044fc5b4c806e03890b5fd3f3eccd39596a2","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-21-msp6cs\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x9c29aff29a0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x9c29aff2900\u003e{\n            observation: \u003c*errors.errorString | 0x9c29c8285c0\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x9c29aff28e0\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-21-msp6cs\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x9c29aff29a0\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x9c29aff2900\u003e{\n            observation: \u003c*errors.errorString | 0x9c29c8285c0\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x9c29aff28e0\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ac84d34217ffed7bf1fb67dcf92fd423dbf5956ae0f518883c8a8dd3cbad89a8","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0"}},{"row_id":"3d8c14d2b94d69244234982b44aaa076734c026be9b1aac8224135fbc79baaa3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","failed_at":"e2e","signature_id":"d12f7ad296c63d2db8d0091c07a0e4b76c50e119a987979054a17bce6c78d0bf","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-22-698b76\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x246f7939e000\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.006s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.006s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x246f7b88a670\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-4-22-698b76\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x246f7939e000\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.006s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.006s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*errors.errorString | 0x246f7b88a670\u003e{\n            s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        },\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ac84d34217ffed7bf1fb67dcf92fd423dbf5956ae0f518883c8a8dd3cbad89a8","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0"}},{"row_id":"42824a6104a0b1eeab200e722b96cbdbbb9eee9e52e7481893dff10c734aacc6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","failed_at":"e2e","signature_id":"6464f6e3e62a79a6ecad40d9da5fa9f7a3ca4e1c274f987a91f46ad0ef5805f7","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3184654d9920\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3184654d9920\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"42e92943fec7b2bbf153cf73ed90a7f4c1a855040fbceb9f58ca92e885cd7190","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","failed_at":"e2e","signature_id":"33db1c3a753f86de5437aeea4ce94aea50eff6efcc9e24481fd3b5a5db91355d","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x9f6ae185170\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x9f6ae185170\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"b51038c496320e294a62815507005afbb6fe240a38acd06b2ada2807df3ffb49","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","failed_at":"e2e","signature_id":"8d94601ad9e8035e94766bfdd3ff32bdad6a27008af0242092dd096fe5ae318d","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2dc96d63c860\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2dc96d63c860\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"d37cbc46467a40b61c4fbc1f4c17b5bf045a4de6f5e9109a76754887aff945eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","failed_at":"e2e","signature_id":"3f2b2137f83a3269a387de589f4ec90449d00621fe948040b817efb69653752e","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x1e14e7538290\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x1e14e7538290\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"eded0aa549cbf14fc925c1288ff53b07f390beda02da5ab61dbe2944b5489306","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","failed_at":"e2e","signature_id":"6aad85da11b23d51d01d7048ab2fcb7dd17e96a24595d2c070d12b0e2654fa14","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x64ca1ba890\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:511]: ACM must not create its ingress certificate secret\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x64ca1ba890\u003e: \n    ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\n    {\n        s: \"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","failure_pattern":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed","search_query":"ACM ingress secret openshift-ingress/cluster-ingress-cert exists; expected ACM certificate delivery to be suppressed"}},{"row_id":"f8cdca68924f8203dd40693cde651f45a224d8b85a906643fafa65226dd35b04","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2113812730220646400","occurred_at":"2026-10-24T02:00:02Z","failed_at":"e2e","signature_id":"48bcb095dab95d4300f51495f479877c48fae6ac937eca31ce9ee317ffaa9c61","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-vsfrhd\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1b75911f3b00\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x1b75911f3ae0\u003e{\n            observation: \u003c*errors.errorString | 0x1b7593506210\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x1b75911f3a60\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: default-ingress-cert must exist and be referenced by IngressController/default for cluster \"cluster-candidate-5-0-vsfrhd\"\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1b75911f3b00\u003e: \n    VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \"default-ingress-cert\"\n    {\n        msg: \"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0.004s: IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n        err: \u003c*verifiers.pollObservationError | 0x1b75911f3ae0\u003e{\n            observation: \u003c*errors.errorString | 0x1b7593506210\u003e{\n                s: \"IngressController/default references \u0026LocalObjectReference{Name:cluster-ingress-cert,}, expected secret \\\"default-ingress-cert\\\"\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x1b75911f3a60\u003e{\n                msg: \"getting default ingress serving certificate secret openshift-ingress/default-ingress-cert: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ac84d34217ffed7bf1fb67dcf92fd423dbf5956ae0f518883c8a8dd3cbad89a8","failure_pattern":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0","search_query":"VerifyDefaultIngressServingCertificateSecretWiring timed out after 10m0"}}],"failure_pattern_summary":{"signature_count":10,"failure_pattern_ids":["0b69f0e7bf323b10963346a5416a79c8f37ccc4ed9e679a584957bb600bac87b","ac84d34217ffed7bf1fb67dcf92fd423dbf5956ae0f518883c8a8dd3cbad89a8"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}