Add code for preparing the new prompt

Author: StephanAkkerman

backend/mnemorai/constants/languages.py

@@ -5,3 +5,156 @@
 with open(config.get("G2P").get("LANGUAGE_JSON")) as f:
     G2P_LANGCODES = json.load(f)
 G2P_LANGUAGES: dict = dict(map(reversed, G2P_LANGCODES.items()))
+
+EPITRAN_LANGCODES = {
+    "aar-Latn": "Afar",
+    "afr-Latn": "Afrikaans",
+    "aii-Syrc": "Assyrian Neo-Aramaic",
+    "amh-Ethi": "Amharic",
+    "amh-Ethi-pp": "Amharic (more phonetic)",
+    "amh-Ethi-red": "Amharic (reduced)",
+    "ara-Arab": "Literary Arabic",
+    "ava-Cyrl": "Avaric",
+    "aze-Cyrl": "Azerbaijani (Cyrillic)",
+    "aze-Latn": "Azerbaijani",
+    "ben-Beng": "Bengali",
+    "ben-Beng-red": "Bengali (reduced)",
+    "ben-Beng-east": "Eastern Bengali",
+    "bho-Deva": "Bhojpuri",
+    "bxk-Latn": "Bukusu",
+    "cat-Latn": "Catalan",
+    "ceb-Latn": "Cebuano",
+    "ces-Latn": "Czech",
+    "cjy-Latn": "Jin (Wiktionary)",
+    "ckb-Arab": "Sorani",
+    "cmn-Hans": "Mandarin (Simplified)*",
+    "cmn-Hant": "Mandarin (Traditional)*",
+    "cmn-Latn": "Mandarin (Pinyin)*",
+    "csb-Latn": "Kashubian",
+    "deu-Latn": "German",
+    "deu-Latn-np": "German†",
+    "deu-Latn-nar": "German (more phonetic)",
+    "eng-Latn": "English",
+    "epo-Latn": "Esperanto",
+    "est-Latn": "Estonian",
+    "fas-Arab": "Farsi (Perso-Arabic)",
+    "fin-Latn": "Finnish",
+    "fra-Latn": "French",
+    "fra-Latn-np": "French†",
+    "fra-Latn-p": "French (more phonetic)",
+    "ful-Latn": "Fulah",
+    "gan-Latn": "Gan (Wiktionary)",
+    "glg-Latn": "Galician",
+    "got-Goth": "Gothic",
+    "got-Latn": "Gothic (Latin)",
+    "hak-Latn": "Hakka (pha̍k-fa-sṳ)",
+    "hat-Latn-bab": "Haitian (Latin-Babel)",
+    "hau-Latn": "Hausa",
+    "hin-Deva": "Hindi",
+    "hmn-Latn": "Hmong",
+    "hrv-Latn": "Croatian",
+    "hsn-Latn": "Xiang (Wiktionary)",
+    "hun-Latn": "Hungarian",
+    "ilo-Latn": "Ilocano",
+    "ind-Latn": "Indonesian",
+    "ita-Latn": "Italian",
+    "jam-Latn": "Jamaican",
+    "jav-Latn": "Javanese",
+    "jpn-Hrgn": "Japanese (Hiragana)",
+    "jpn-Hrgn-red": "Japanese (Hiragana, reduced)",
+    "jpn-Ktkn": "Japanese (Katakana)",
+    "jpn-Ktkn-red": "Japanese (Katakana, reduced)",
+    "jpn-Jpan": "Japanese (Hiragana, Katakana, Kanji)",
+    "jpn-Hira": "Japanese (Hiragana)",
+    "jpn-Hira-red": "Japanese (Hiragana, reduced)",
+    "jpn-Kana": "Japanese (Katakana)",
+    "jpn-Kana-red": "Japanese (Katakana, reduced)",
+    "kat-Geor": "Georgian",
+    "kaz-Cyrl": "Kazakh (Cyrillic)",
+    "kaz-Cyrl-bab": "Kazakh (Cyrillic—Babel)",
+    "kaz-Latn": "Kazakh (Latin)",
+    "kbd-Cyrl": "Kabardian",
+    "khm-Khmr": "Khmer",
+    "kin-Latn": "Kinyarwanda",
+    "kir-Arab": "Kyrgyz (Perso-Arabic)",
+    "kir-Cyrl": "Kyrgyz (Cyrillic)",
+    "kir-Latn": "Kyrgyz (Latin)",
+    "kmr-Latn": "Kurmanji",
+    "kmr-Latn-red": "Kurmanji (reduced)",
+    "kor-Hang": "Korean",
+    "lao-Laoo": "Lao",
+    "lao-Laoo-prereform": "Lao (Before spelling reform)",
+    "lav-Latn": "Latvian",
+    "lez-Cyrl": "Lezgian",
+    "lij-Latn": "Ligurian",
+    "lit-Latn": "Lithuanian",
+    "lsm-Latn": "Saamia",
+    "ltc-Latn-bax": "Middle Chinese (Baxter and Sagart 2014)",
+    "lug-Latn": "Ganda / Luganda",
+    "mal-Mlym": "Malayalam",
+    "mar-Deva": "Marathi",
+    "mlt-Latn": "Maltese",
+    "mon-Cyrl-bab": "Mongolian (Cyrillic)",
+    "mri-Latn": "Maori",
+    "msa-Latn": "Malay",
+    "mya-Mymr": "Burmese",
+    "nan-Latn": "Hokkien (pe̍h-oē-jī)",
+    "nan-Latn-tl": "Hokkien (Tâi-lô)",
+    "nld-Latn": "Dutch",
+    "nya-Latn": "Chichewa",
+    "ood-Latn-alv": "Tohono O'odham (Alvarez-Hale)",
+    "ood-Latn-sax": "Tohono O'odham (Saxton)",
+    "ori-Orya": "Odia",
+    "orm-Latn": "Oromo",
+    "pan-Guru": "Punjabi (Eastern)",
+    "pol-Latn": "Polish",
+    "por-Latn": "Portuguese",
+    "quy-Latn": "Ayacucho Quechua / Quechua Chanka",
+    "ron-Latn": "Romanian",
+    "run-Latn": "Rundi",
+    "rus-Cyrl": "Russian",
+    "sag-Latn": "Sango",
+    "sin-Sinh": "Sinhala",
+    "slv-Latn": "Slovene / Slovenian",
+    "sna-Latn": "Shona",
+    "som-Latn": "Somali",
+    "spa-Latn": "Spanish",
+    "spa-Latn-eu": "Spanish (Iberian)",
+    "sqi-Latn": "Albanian",
+    "sro-Latn": "Sardinian (Campidanese)",
+    "srp-Latn": "Serbian (Latin)",
+    "srp-Cyrl": "Serbian (Cyrillic)",
+    "swa-Latn": "Swahili",
+    "swa-Latn-red": "Swahili (reduced)",
+    "swe-Latn": "Swedish",
+    "tam-Taml": "Tamil",
+    "tam-Taml-red": "Tamil (reduced)",
+    "tel-Telu": "Telugu",
+    "tgk-Cyrl": "Tajik",
+    "tgl-Latn": "Tagalog",
+    "tgl-Latn-red": "Tagalog (reduced)",
+    "tha-Thai": "Thai",
+    "tir-Ethi": "Tigrinya",
+    "tir-Ethi-pp": "Tigrinya (more phonemic)",
+    "tir-Ethi-red": "Tigrinya (reduced)",
+    "tok-Latn": "Toki Pona",
+    "tpi-Latn": "Tok Pisin",
+    "tuk-Cyrl": "Turkmen (Cyrillic)",
+    "tuk-Latn": "Turkmen (Latin)",
+    "tur-Latn": "Turkish (Latin)",
+    "tur-Latn-bab": "Turkish (Latin—Babel)",
+    "tur-Latn-red": "Turkish (reduced)",
+    "ukr-Cyrl": "Ukrainian",
+    "urd-Arab": "Urdu",
+    "uig-Arab": "Uyghur (Perso-Arabic)",
+    "uzb-Cyrl": "Uzbek (Cyrillic)",
+    "uzb-Latn": "Uzbek (Latin)",
+    "vie-Latn": "Vietnamese",
+    "wuu-Latn": "Shanghainese Wu (Wiktionary)",
+    "xho-Latn": "Xhosa",
+    "yor-Latn": "Yoruba",
+    "yue-Latn": "Cantonese (Jyutping)",
+    "yue-Hant": "Cantonese (Character)",
+    "zha-Latn": "Zhuang",
+    "zul-Latn": "Zulu",
+}

backend/mnemorai/utils/bigrammer.py

@@ -0,0 +1,77 @@
+from collections import Counter
+from itertools import chain
+
+from nltk import bigrams, download
+from nltk.corpus import brown, gutenberg, reuters, webtext
+from wordfreq import zipf_frequency
+
+# Download the necessary NLTK corpora if not already present
+for c in ["reuters", "gutenberg", "webtext"]:
+    download(c, quiet=True)
+
+# Zipf → raw unigram probability.  Zipf 6 ≈ 1/1 000, so P = 10**(zipf−9)
+_unigram_p = lambda w: 10 ** (zipf_frequency(w, "en") - 9)
+
+tokens = [
+    w.lower()
+    for w in chain(brown.words(), reuters.words(), gutenberg.words(), webtext.words())
+]
+_unigram_cnt = Counter(tokens)
+_bigram_cnt = Counter(bigrams(tokens))
+_V = len(_unigram_cnt)
+
+
+def backoff_prob(w1: str, w2: str) -> float:
+    """Naïve product P(w1)·P(w2) from the Wordfreq unigram model."""
+    return _unigram_p(w1) * _unigram_p(w2)
+
+
+def brown_bigram_prob(w1: str, w2: str) -> float:
+    # Laplace-smoothed P(w2 | w1)
+    return (_bigram_cnt[(w1, w2)] + 1) / (_unigram_cnt[w1] + _V)
+
+
+def bigram_prob(w1: str, w2: str, *, alpha: float = 0.1) -> float:
+    """
+    Hybrid probability.
+
+      • if Brown corpus has seen (w1,w2), return its Laplace value
+      • otherwise fall back to alpha·P_wordfreq(w1)·P_wordfreq(w2)
+
+    `alpha` (default 0.1) keeps back-off numbers on the same scale as
+    real bigram counts—tune it if the gap feels too big or small.
+    """
+    base = brown_bigram_prob(w1, w2)
+    if _bigram_cnt[(w1, w2)] == 0:  # unseen → use back-off
+        return alpha * backoff_prob(w1, w2)
+    return base
+
+
+def bigram_grid(list1, list2, *, sort_desc: bool = True):
+    """Return (w1, w2, hybrid-prob) for every w1∈list1, w2∈list2."""
+    out = [
+        (w1, w2, bigram_prob(w1.lower(), w2.lower()))  # ← CHANGED
+        for w1 in list1
+        for w2 in list2
+    ]
+    return sorted(out, key=lambda t: t[2], reverse=sort_desc) if sort_desc else out
+
+
+if __name__ == "__main__":
+    unformatted_list1 = (
+        "duh, ta, tea, toe, tie, dew, due, doe, dough, die, dart, door, "
+        "thaw, though, there, tire, dare, donor, draw, tear, door, data, "
+        "deter, tune, the"
+    )
+    unformatted_list2 = (
+        "sing, sink, sting, thing, wing, king, cling, grin, gin, gang, gone, "
+        "gong, then, thens, ten, tang, tan, town, tongue, tinge, begin, bing, "
+        "singe, swing, twin"
+    )
+    list1 = unformatted_list1.split(", ")
+    list2 = unformatted_list2.split(", ")
+
+    for w1, w2, p in bigram_grid(list1, list2):
+        print(f"{w1} {w2:<10}  P={p:.3e}")
+
+# Maybe also do levenshtein distance between for the chunks