Search that still works when nobody types tone marks

import unicodedata
from olaverse.nlp import Diacritizer

# Your corpus language is something you know — don't infer it from stripped text
d = Diacritizer(model="diacnet-1.0", lang="yo")

def fold(s: str) -> str:
    """Accent-fold to a match key. Pure Unicode, no model — ọ/ẹ/ṣ decompose to
       o/e/s plus a combining mark, so NFD and dropping Mn covers Yoruba."""
    return "".join(c for c in unicodedata.normalize("NFD", s)
                   if unicodedata.category(c) != "Mn").lower()

def index_document(doc):
    # Restore only what's missing its marks; never re-run it on correct text
    text = doc["text"] if doc["has_diacritics"] else d.restore(doc["text"])
    return {**doc, "display_text": text, "match_text": fold(text)}

corpus = [
    {"id": 1, "text": "Ṣé ẹranko náà sì gbọ́ ọ?", "has_diacritics": True},    # clean CMS copy
    {"id": 2, "text": "se eranko naa si gbo o?",  "has_diacritics": False},   # scraped / OCR
]
indexed = [index_document(c) for c in corpus]
# both now share one match_text: 'se eranko naa si gbo o?'
# and doc 2 displays as 'ṣé ẹranko náà sì gbọ́ ọ?' instead of the stripped original

# The query path is folding only — no model, no added latency
key = fold("SE ERANKO NAA")
[c["id"] for c in indexed if key in c["match_text"]]     # → [1, 2]