add models

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+embedding_data.csv filter=lfs diff=lfs merge=lfs -text
+embeddings.npy filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,116 @@
+#%%
+import gradio as gr
+from dotenv import load_dotenv
+
+from translate import run_translate
+from diacritize import diacritize, diacritize_if_not_already
+from translit import taatik
+from semsearch import update_df
+from tts import get_audio
+from consts import CSS, ABOUT, JS_FUNC
+load_dotenv()
+
+with gr.Blocks(title = "Levanti - כלי תרגום לערבית מדוברת",
+               css=CSS,
+               theme="default") as demo:
+    # gr.HTML("<h2><span style='color: #2563eb'>Levantine Arabic</span> Translator</h2>")
+    gr.HTML("<h2><span dir='rtl'><span style='color: #2563eb'>Levanti</span>ne Translator</span></h2>כלי תרגום לערבית מדוברת")
+    with gr.Tab('תרגום', elem_id="tab1"):
+        with gr.Row():
+            with gr.Column():
+                input_text = gr.Textbox(label="קלט",
+                                        info = "עברית או ערבית מדוברת",
+                                        placeholder="הזינו טקסט בערבית או עברית",
+                                        lines=2,
+                                        elem_id="input",
+                                        rtl=True) 
+
+                gr.Examples(["רציתי ללכת אתמול לחנות, אבל ירד גשם", "خلينا ندور على مطعم تاني"],
+                            input_text,
+                            label="דוגמאות")
+
+                btn = gr.Button("תרגום")
+                with gr.Row():
+                    dialect = gr.Radio(["פלסטיני", "סורי", "לבנוני", "מצרי"],
+                                        label = "להג",
+                                        info="משפיע על תרגום לערבית",
+                                        value="פלסטיני")
+
+                # gr.Markdown("Built by [Guy Mor-Lan](mailto:guy.mor@mail.huji.ac.il). Pronunciation model is specifically tailored to urban Palestinian Arabic. Text-to-speech uses Microsoft Azure's API and may provide different result from the transliterated pronunciation.")
+                gr.Markdown("נוצר על ידי [גיא מור-לן](mailto:guy.mor@mail.huji.ac.il) כחלק מפרויקט [Levanti](https://huggingface.co/datasets/guymorlan/levanti). השמע מופק באמצעות Azure TTS על בסיס הניקוד המנובא ויוריסטיקות.", elem_id="footer", rtl=True)
+
+            with gr.Column():
+                with gr.Group(elem_id="grp"):
+                        gr.HTML("<div dir='rtl'>תרגום</div>")
+                        # gr.Markdown("תרגום", elem_id="diacritized")
+                        translation_output = gr.HTML("<br>", visible=True, label="תרגום", elem_id="main")
+
+                hidden_arabic = gr.Textbox(lines=1, elem_id="trans", visible=False)
+
+                diacritized_output = gr.Textbox(label="ניקוד (ניסיוני)", lines=1, elem_id="diacritized",
+                                                 rtl=True, interactive=False)
+                taatik_output = gr.Textbox(label="תעתיק (ניסיוני)", lines=1, elem_id="taatik",
+                                           text_align="right", rtl=True, interactive=False)
+                # diacritized_output = gr.HTML("<br>", label="ניקוד")
+                # taatik_output = gr.HTML("<br>", label="תעתיק")
+
+                audio = gr.Audio(label="שמע (Azure)", interactive=False,
+                                 autoplay=True)
+                audio_button = gr.Button("צור שמע")
+                audio_button.click(get_audio, inputs=[diacritized_output], outputs=[audio])
+
+
+        btn.click(run_translate, inputs=[input_text, dialect],
+                  outputs=[translation_output, hidden_arabic], api_name="en2ar",
+                  js="function jump(x, y){document.getElementById('main').scrollIntoView(); return [x, y];}")
+
+        input_text.submit(run_translate, inputs=[input_text, dialect],
+                          outputs=[translation_output, hidden_arabic], scroll_to_output=True)
+        hidden_arabic.change(diacritize, inputs=[hidden_arabic], outputs=[diacritized_output])
+        diacritized_output.change(taatik, inputs=[diacritized_output], outputs=[taatik_output])
+        # pal.change(get_transliteration, inputs=[pal, include], outputs=[pal_translit]);
+        # include.change(toggle_visibility, inputs=[include], outputs=[pal_translit, sy, lb, eg])
+        with gr.Row():
+            # import pandas as pd
+            # ex_df = pd.DataFrame({"text": ["שלום", "כיצד ניתן לעזור לך?", "איפה נמצא המסעדה הכי טובה בעיר?"]})
+            # ex_df = ex_df.style.set_properties(**{
+            #     'font-family': 'Arial, sans-serif',
+            #     'text-align': 'right'
+            # })
+            # bla_df = gr.DataFrame(ex_df, visible=True, elem_id="nearest", wrap =True)
+            nearest_df = gr.DataFrame(headers=["ערבית", "עברית", "מאומת"], visible=False, wrap=True,
+                                      elem_id="nearest", label="תוצאות קרובות מתוך קורפוס Levanti", height=300)
+
+        hidden_arabic.change(update_df, inputs=[hidden_arabic], outputs=[nearest_df])
+        
+    with gr.Tab("ניקוד ותעתוק", elem_id="tab2"):
+        with gr.Row():
+            with gr.Column():
+                diac_text = gr.Textbox(label="קלט", placeholder="הזינו טקסט בערבית", lines=1,
+                                        info = "בשביל תעתוק בלבד, הזינו טקסט ערבי מנוקד",
+                                        elem_id="diac_input", rtl=True)
+                gr.Examples(["خلينا ندور على مطعم تاني", "قَدِيْش حَقّ الْبَنْدُورَة؟"], diac_text, 
+                            label="דוגמאות", elem_id="diac_ex")
+                btn2 = gr.Button("שליחה")
+                
+            with gr.Column():
+                diacritized_output2 = gr.Textbox(label="ניקוד", lines=1,
+                                                  elem_id="diacritized2", rtl=True)
+                taatik_output2 = gr.Textbox(label="תעתיק", lines=1,
+                                             elem_id="taatik2", rtl=True)
+
+        # input_text.submit(run_translate, inputs=[input_text, dialect],
+        #                   outputs=[translation_output], scroll_to_output=True)
+        # hidden_arabic.change(diacritize, inputs=[hidden_arabic], outputs=[diacritized_output])
+        # diacritized_output.change(taatik, inputs=[diacritized_output], outputs=[taatik_output])
+        btn2.click(diacritize_if_not_already, inputs=[diac_text], outputs=[diacritized_output2])
+        diac_text.submit(diacritize_if_not_already, inputs=[diac_text], outputs=[diacritized_output2])
+        diacritized_output2.change(taatik, inputs=[diacritized_output2], outputs=[taatik_output2])
+    with gr.Tab("אודות", elem_id="tab3"):
+        with gr.Row():
+            gr.HTML("<h2>אודות</h2>")
+            gr.Markdown(ABOUT, elem_id="about", rtl=True)
+
+
+
+demo.launch(ssl_verify=False)

ar_he/README.md

@@ -0,0 +1,32 @@
+---
+license: cc-by-nc-4.0
+datasets:
+- guymorlan/levanti
+language:
+- ar
+- he
+pipeline_tag: translation
+widget:
+- text: بدي أروح ع الدكان بكرا
+---
+
+# Levanti (colloquial Levantine Arabic -> Hebrew) translator
+
+Trained on the [Levanti](https://huggingface.co/datasets/guymorlan/levanti) dataset by fine-tuning [Helsinki-NLP/opus-mt-ar-he](https://huggingface.co/Helsinki-NLP/opus-mt-ar-he) for 8 epochs.
+The model supports Palestinian, Jordanian, Syrian, Lebanese and Egyptian dialects.
+
+
+# Example usage
+
+```python
+from transformers import pipeline
+trans = pipeline("translation", "guymorlan/levanti_translate_ar_he")
+trans("بدي أروح ع الدكان بكرا")
+```
+```
+Out[1]: [{'translation_text': 'אני רוצה ללכת לחנות מחר'}]
+```
+
+# Attribution
+Created by Guy Mor-Lan.<br>
+Contact: guy.mor AT mail.huji.ac.il

ar_he/config.json

@@ -0,0 +1,61 @@
+{
+  "_name_or_path": "Helsinki-NLP/opus-mt-ar-he",
+  "activation_dropout": 0.0,
+  "activation_function": "swish",
+  "add_bias_logits": false,
+  "add_final_layer_norm": false,
+  "architectures": [
+    "MarianMTModel"
+  ],
+  "attention_dropout": 0.0,
+  "bad_words_ids": [
+    [
+      63333
+    ]
+  ],
+  "bos_token_id": 0,
+  "classif_dropout": 0.0,
+  "classifier_dropout": 0.0,
+  "d_model": 512,
+  "decoder_attention_heads": 8,
+  "decoder_ffn_dim": 2048,
+  "decoder_layerdrop": 0.0,
+  "decoder_layers": 6,
+  "decoder_start_token_id": 63333,
+  "decoder_vocab_size": 63334,
+  "dropout": 0.1,
+  "encoder_attention_heads": 8,
+  "encoder_ffn_dim": 2048,
+  "encoder_layerdrop": 0.0,
+  "encoder_layers": 6,
+  "eos_token_id": 0,
+  "extra_pos_embeddings": 63334,
+  "forced_eos_token_id": 0,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2"
+  },
+  "init_std": 0.02,
+  "is_encoder_decoder": true,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_2": 2
+  },
+  "max_length": 512,
+  "max_position_embeddings": 512,
+  "model_type": "marian",
+  "normalize_before": false,
+  "normalize_embedding": false,
+  "num_beams": 4,
+  "num_hidden_layers": 6,
+  "pad_token_id": 63333,
+  "scale_embedding": true,
+  "share_encoder_decoder_embeddings": true,
+  "static_position_embeddings": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.38.1",
+  "use_cache": true,
+  "vocab_size": 63334
+}

ar_he/generation_config.json

@@ -0,0 +1,16 @@
+{
+  "bad_words_ids": [
+    [
+      63333
+    ]
+  ],
+  "bos_token_id": 0,
+  "decoder_start_token_id": 63333,
+  "eos_token_id": 0,
+  "forced_eos_token_id": 0,
+  "max_length": 512,
+  "num_beams": 4,
+  "pad_token_id": 63333,
+  "renormalize_logits": true,
+  "transformers_version": "4.38.1"
+}

ar_he/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50672992291e20bc714b766615195a72d09b3950362adc5a0097b6bbfc5b630d
+size 306544408

ar_he/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

ar_he/tokenizer_config.json

@@ -0,0 +1,39 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "63333": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</s>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "return_tensors": "pt",
+  "separate_vocabs": false,
+  "source_lang": "ara",
+  "sp_model_kwargs": {},
+  "target_lang": "heb",
+  "tokenizer_class": "MarianTokenizer",
+  "unk_token": "<unk>"
+}

ar_he_ct2/config.json

@@ -0,0 +1,10 @@
+{
+  "add_source_bos": false,
+  "add_source_eos": false,
+  "bos_token": "<s>",
+  "decoder_start_token": "</s>",
+  "eos_token": "</s>",
+  "layer_norm_epsilon": null,
+  "multi_query_attention": false,
+  "unk_token": "<unk>"
+}

ar_he_ct2/model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa9558eb1d4c737e67e2d8238eb5c56e15721911cf75a23e8906225cc417c58a
+size 307573250

colorize.py

@@ -0,0 +1,109 @@
+import matplotlib.pyplot as plt
+import numpy as np
+
+
+def generate_diverging_colors(num_colors, palette='Set3'): # courtesy of ChatGPT
+    # Generate a colormap with a specified number of colors
+    cmap = plt.cm.get_cmap(palette, num_colors)
+
+    # Get the RGB values of the colors in the colormap
+    colors_rgb = cmap(np.arange(num_colors))
+
+    # Convert the RGB values to hexadecimal color codes
+    colors_hex = [format(int(color[0]*255)<<16|int(color[1]*255)<<8|int(color[2]*255), '06x') for color in colors_rgb]
+
+    return colors_hex
+
+
+def align_words(outputs, tokenizer, encoder_input_ids, decoder_input_ids,
+                threshold=0.4, skip_first_src=True, skip_second_src=False,
+                layer=2, head=6):
+
+    alignment = []
+    # threshold = 0.05
+    for i, tok in enumerate(outputs.cross_attentions[layer][0][head]):
+        alignment.append([[i], (tok > threshold).nonzero().squeeze(-1).tolist()])
+
+    # for i in alignment:
+    #     src_tok = [tokenizer.decode(decoder_input_ids[0][x]) for x in i[0]]
+    #     trg_tok = [tokenizer.decode(encoder_input_ids[0][x]) for x in i[1]]
+    #     print(src_tok, "=>", trg_tok)
+
+    merged = []
+    for i in alignment:
+        token = tokenizer.convert_ids_to_tokens([decoder_input_ids[0][i[0]]])[0]
+        # print(token)
+        if token not in ["</s>", "<pad>", "<unk>", "<s>"]:
+            if merged:
+                tomerge = False
+                # check overlap with previous entry
+                for x in i[1]:
+                    if x in merged[-1][1]:# or tokenizer.convert_ids_to_tokens([encoder_input_ids[0][x]])[0][0] != "▁": 
+                        tomerge = True
+                        break
+                # if first character is not a "▁"
+                if token[0] != "▁":
+                    tomerge = True
+                if tomerge:
+                    merged[-1][0] += i[0]
+                    merged[-1][1] += i[1]
+                else:
+                    merged.append(i)
+            else:
+                merged.append(i)
+                
+    # print("=====MERGED=====")
+    # for i in merged:
+    #     src_tok = [tokenizer.decode(decoder_input_ids[0][x]) for x in i[0]]
+    #     trg_tok = [tokenizer.decode(encoder_input_ids[0][x]) for x in i[1]]
+    #     print(src_tok, "=>", trg_tok)
+
+    colordict = {}
+    ncolors = 0
+    for i in merged:
+        src_tok = [f"src_{x}" for x in i[0]]
+        trg_tok = [f"trg_{x}" for x in i[1]]
+        all_tok = src_tok + trg_tok
+        # see if any tokens in entry already have associated color
+        newcolor = None
+        for t in all_tok:
+            if t in colordict:
+                newcolor = colordict[t]
+                break
+        if not newcolor:
+            newcolor = ncolors
+            ncolors += 1
+        for t in all_tok:
+            if t not in colordict:
+                colordict[t] = newcolor
+
+    colors = generate_diverging_colors(ncolors, palette="Set2")
+    id_to_color = {i: c for i, c in enumerate(colors)}
+    for k, v in colordict.items():
+        colordict[k] = id_to_color[v]
+
+    tgthtml = []
+    for i, token in enumerate(decoder_input_ids[0]):
+        if f"src_{i}" in colordict:
+            label = f"src_{i}"
+            tgthtml.append(f"<span style='color: #{colordict[label]}'>{tokenizer.convert_ids_to_tokens([token])[0]}</span>")
+        else:
+            tgthtml.append(f"<span style='color: --color-text-body'>{tokenizer.convert_ids_to_tokens([token])[0]}</span>")
+    tgthtml = "".join(tgthtml)
+    tgthtml = tgthtml.replace("▁", " ")
+    tgthtml = f"<span style='font-size: 25px'>{tgthtml}</span>"
+
+    srchtml = []
+    for i, token in enumerate(encoder_input_ids[0]):
+        if  (i == 0 and skip_first_src) or (i == 1 and skip_second_src):
+            continue
+
+        if f"trg_{i}" in colordict:
+            label = f"trg_{i}"
+            srchtml.append(f"<span style='color: #{colordict[label]}'>{tokenizer.convert_ids_to_tokens([token])[0]}</span>")
+        else:
+            srchtml.append(f"<span style='color: --color-text-body'>{tokenizer.convert_ids_to_tokens([token])[0]}</span>")
+    srchtml = "".join(srchtml)
+    srchtml = srchtml.replace("▁", " ")
+    srchtml = f"<span style='font-size: 25px'>{srchtml}</span>"
+    return srchtml, tgthtml

consts.py

@@ -0,0 +1,114 @@
+CSS = """
+
+@import url('https://fonts.googleapis.com/css2?family=Noto+Sans+Hebrew:wght@400;700&family=Noto+Naskh+Arabic:wght@400;700&display=swap');
+
+#taatik textarea { 
+    font-size: 25px;
+    font-family: 'Noto Sans Hebrew', 'Noto Naskh Arabic', 'SBL Hebrew', 'David CLM', 'FrankRuehl CLM', 'Narkisim', 'Arial', 'Arial Unicode MS', sans-serif;
+}
+
+@font-face {
+    font-family: 'Noto Sans Hebrew';
+    src: url('https://fonts.gstatic.com/s/notosanshebrew/v40/or3HQ7v33eiDlKj4557q0OGCZa662.woff2') format('woff2');
+    unicode-range: U+0590-05FF, U+200C-2010, U+20AA, U+25CC, U+FB1D-FB4F;
+}
+
+@font-face {
+    font-family: 'Noto Naskh Arabic';
+    src: url('https://fonts.gstatic.com/s/notonaskharabic/v30/RrQ5bpV-9Dd1b1OAGA6M9PkyDuVBePeKNaxcsss0Y7bwvc5Urqjc.woff2') format('woff2');
+    unicode-range: U+0600-06FF, U+0750-077F, U+0870-088E, U+0890-0891, U+0898-08E1, U+08E3-08FF, U+200C-200E, U+2010-2011, U+204F, U+2E41, U+FB50-FDFF, U+FE70-FE74, U+FE76-FEFC;
+}
+
+:root { direction: rtl; }
+#liter textarea, #trans textarea { font-size: 25px;}
+#grp { padding: 10px; }
+#trans textarea { direction: rtl; }
+#taatik { direction: rtl; }
+#about { direction: rtl; }
+#tab1 { direction: rtl; }
+#tab2 { direction: rtl; }
+#footer { direction: rtl; }
+#input {direction: rtl;}
+#diac_input {direction: rtl;}
+#diacritized { direction: rtl; }
+#diacritized2 { direction: rtl; }
+#taatik2 { direction: rtl; }
+#diacritized textarea { font-size: 25px;}
+#diacritized2 textarea { font-size: 25px;}
+#taatik2 textarea { font-size: 25px;}
+#input textarea { font-size: 20px;}
+#diac_input textarea { font-size: 20px;}
+#check { border-style: none !important; }
+#nearest { font-family: 'SBL Hebrew', 'David CLM', 'FrankRuehl CLM', 'Narkisim', 'Arial'; }
+:root {--button-secondary-background-focus: #2563eb !important;
+       --button-secondary-background-base: #2563eb !important;
+       --button-secondary-background-hover: linear-gradient(to bottom right, #0692e8, #5859c2);
+       --button-secondary-text-color-base: white !important;
+       --button-secondary-text-color-hover: white !important;
+       --button-secondary-background-focus: rgb(51 122 216 / 70%) !important;
+       --button-secondary-text-color-focus: white !important}
+.dark {--button-secondary-background-base: #2563eb !important;
+       --button-secondary-background-focus: rgb(51 122 216 / 70%) !important;
+       --button-secondary-background-hover: linear-gradient(to bottom right, #0692e8, #5859c2)}
+.feather-music { stroke: #2563eb; }
+
+.dataframe {
+    font-family: 'Arial', 'Helvetica', sans-serif !important;
+}
+.dataframe th, .dataframe td {
+    font-family: inherit !important;
+
+}
+
+.gradio-container .dataframe {
+    font-family: Arial, sans-serif !important;
+
+}
+
+/* Target the table header cells */
+.table th .cell-wrap {
+    text-align: right !important;
+}
+
+/* Target the span inside the header cells */
+.table th .cell-wrap span {
+    text-align: right !important;
+    display: block;
+    font-family: Arial, sans-serif !important;
+}
+
+/* Ensure the sort button doesn't interfere with alignment */
+.table th .cell-wrap .sort-button {
+    float: left;
+}
+
+/* Target the table body cells */
+.table td {
+    text-align: right !important;
+}
+
+/* Target the span inside the body cells */
+.table td .cell-wrap span {
+    text-align: right !important;
+    display: block;
+    font-family: Arial, sans-serif !important;
+    font-size: 20px;
+}
+
+"""
+
+ABOUT = """
+כלי זה נוצר על ידי גיא מור-לן כחלק מפרויקט Levanti. הכלי מתרגם טקסט בעברית לדיאלקטים השונים של ערבית מדוברת, ולהפך, באמצעות מודלים יעודיים של בינה מלאכותית. כמו כן הכלי תומך בניקוד אוטומטי, המרה של הניקוד לתעתוק והפקת שמע (פיצ'רים ניסיוניים). לפרטים נוספים, גישה לדאטה ולמודלים, ולגרסה האנגלית של כלי ראו את [דף הפרויקט בהאגינג פייס](https://huggingface.co/datasets/guymorlan/levanti).
+מעוניינים לתרום לפרויקט? מצאתם טעות? מוזמנים ליצור קשר [כאן](mailto:guy.mor@mail.huji.ac.il). מוזמנים לבדוק גם את אתר האחות [סרטונים בערבית](https://videosinarabic.com/).
+"""
+
+JS_FUNC = """
+function refresh() {
+    const url = new URL(window.location);
+
+    if (url.searchParams.get('__theme') !== 'dark') {
+        url.searchParams.set('__theme', 'dark');
+        window.location.href = url.href;
+    }
+}
+"""

diacritize.py

@@ -0,0 +1,42 @@
+#%%
+from transformers import RobertaForTokenClassification, AutoTokenizer
+model = RobertaForTokenClassification.from_pretrained("guymorlan/levanti_arabic2diacritics")
+tokenizer = AutoTokenizer.from_pretrained("guymorlan/levanti_arabic2diacritics")
+
+#%%
+label2diacritic = {0: 'ّ', # SHADDA
+                   1: 'َ', # FATHA
+                   2: 'ِ', # KASRA
+                   3: 'ُ', # DAMMA
+                   4: 'ْ'} # SUKKUN
+
+
+def arabic2diacritics(text, model, tokenizer):
+    tokens = tokenizer(text, return_tensors="pt")
+    preds = (model(**tokens).logits.sigmoid() > 0.5)[0][1:-1] # remove preds for BOS and EOS
+    new_text = []
+    for p, c in zip(preds, text):
+        new_text.append(c)
+        for i in range(1, 5):
+            if p[i]:
+                new_text.append(label2diacritic[i])
+        # check shadda last
+        if p[0]:
+            new_text.append(label2diacritic[0])
+        
+    new_text = "".join(new_text)
+    return new_text
+
+
+def diacritize(text):
+    return arabic2diacritics(text, model, tokenizer)
+
+def diacritize_if_not_already(text):
+    if any(c in label2diacritic.values() for c in text):
+        return text
+    else:
+        return arabic2diacritics(text, model, tokenizer)
+#%%
+# text = "بديش اروح عالمدرسة بكرا"
+# arabic2diacritics(text, model, tokenizer)
+# %%

embedding_data.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2819479f36c2c7667febd5745f148c1c82b2c691ef1e04defec96e9c70c7b71b
+size 88125517

embeddings.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08d31408285b68d57a39850295e26e616291065ca3953fb6db9494e0b66ae61c
+size 319545472

he_ar/README.md

@@ -0,0 +1,35 @@
+---
+license: cc-by-nc-4.0
+datasets:
+- guymorlan/levanti
+language:
+- ar
+- he
+pipeline_tag: translation
+widget:
+- text: P אני רוצה ללכת מחר לחנות
+---
+
+# Levanti (Hebrew -> colloquial Levantine Arabic) translator
+
+Trained on the [Levanti](https://huggingface.co/datasets/guymorlan/levanti) dataset by fine-tuning [Helsinki-NLP/opus-mt-he-ar](https://huggingface.co/Helsinki-NLP/opus-mt-ar-he) for 8 epochs.
+This model is trained to support dialect conditional generation by utilizing the first token (followed by a space) as an indicator of the desired dialect:
+* **P** for Palestinian
+* **L** for Lebanese
+* **S** for Syrian
+* **E** for Egyptian
+
+# Example usage
+
+```python
+from transformers import pipeline
+trans = pipeline("translation", "guymorlan/levanti_translate_he_ar")
+trans("P אני רוצה ללכת מחר לחנות")
+```
+```
+Out[1]: [{'translation_text': 'بدي أروح ع الدكان بكرا'}]
+```
+
+# Attribution
+Created by Guy Mor-Lan.<br>
+Contact: guy.mor AT mail.huji.ac.il

he_ar/config.json

@@ -0,0 +1,61 @@
+{
+  "_name_or_path": "Helsinki-NLP/opus-mt-he-ar",
+  "activation_dropout": 0.0,
+  "activation_function": "swish",
+  "add_bias_logits": false,
+  "add_final_layer_norm": false,
+  "architectures": [
+    "MarianMTModel"
+  ],
+  "attention_dropout": 0.0,
+  "bad_words_ids": [
+    [
+      63303
+    ]
+  ],
+  "bos_token_id": 0,
+  "classif_dropout": 0.0,
+  "classifier_dropout": 0.0,
+  "d_model": 512,
+  "decoder_attention_heads": 8,
+  "decoder_ffn_dim": 2048,
+  "decoder_layerdrop": 0.0,
+  "decoder_layers": 6,
+  "decoder_start_token_id": 63303,
+  "decoder_vocab_size": 63304,
+  "dropout": 0.1,
+  "encoder_attention_heads": 8,
+  "encoder_ffn_dim": 2048,
+  "encoder_layerdrop": 0.0,
+  "encoder_layers": 6,
+  "eos_token_id": 0,
+  "extra_pos_embeddings": 63304,
+  "forced_eos_token_id": 0,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1",
+    "2": "LABEL_2"
+  },
+  "init_std": 0.02,
+  "is_encoder_decoder": true,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1,
+    "LABEL_2": 2
+  },
+  "max_length": 512,
+  "max_position_embeddings": 512,
+  "model_type": "marian",
+  "normalize_before": false,
+  "normalize_embedding": false,
+  "num_beams": 4,
+  "num_hidden_layers": 6,
+  "pad_token_id": 63303,
+  "scale_embedding": true,
+  "share_encoder_decoder_embeddings": true,
+  "static_position_embeddings": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.38.1",
+  "use_cache": true,
+  "vocab_size": 63304
+}

he_ar/generation_config.json

@@ -0,0 +1,16 @@
+{
+  "bad_words_ids": [
+    [
+      63303
+    ]
+  ],
+  "bos_token_id": 0,
+  "decoder_start_token_id": 63303,
+  "eos_token_id": 0,
+  "forced_eos_token_id": 0,
+  "max_length": 512,
+  "num_beams": 4,
+  "pad_token_id": 63303,
+  "renormalize_logits": true,
+  "transformers_version": "4.38.1"
+}

he_ar/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:334b98a0c0db5cd649d23521bf3c7ce41d92238092c7b90811434961390ab7b2
+size 306482848

he_ar/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

he_ar/tokenizer_config.json

@@ -0,0 +1,39 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "63303": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</s>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "return_tensors": "pt",
+  "separate_vocabs": false,
+  "source_lang": "heb",
+  "sp_model_kwargs": {},
+  "target_lang": "ara",
+  "tokenizer_class": "MarianTokenizer",
+  "unk_token": "<unk>"
+}

he_ar_ct2/config.json

@@ -0,0 +1,10 @@
+{
+  "add_source_bos": false,
+  "add_source_eos": false,
+  "bos_token": "<s>",
+  "decoder_start_token": "</s>",
+  "eos_token": "</s>",
+  "layer_norm_epsilon": null,
+  "multi_query_attention": false,
+  "unk_token": "<unk>"
+}

he_ar_ct2/model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2ca03c089d8803b9963cfca86b1a760cb56e1f3f776743d21b33aa747bc94e8
+size 307511690

requirements.txt

@@ -0,0 +1,10 @@
+transformers==4.42.3
+torch==1.13.1
+sentencepiece==0.1.97
+sacremoses==0.0.53
+pandas==1.5.1
+azure-cognitiveservices-speech==1.38.0
+matplotlib==3.7.0
+python-dotenv
+gradio==4.37.2
+ctranslate2==4.1.0

semsearch.py

@@ -0,0 +1,78 @@
+import numpy as np
+import torch
+import pandas as pd
+import translate
+import gradio as gr
+
+data = pd.read_csv("./embedding_data.csv")
+embeddings = np.load("./embeddings.npy")
+
+def normalize_vector(v):
+    norm = np.linalg.norm(v)
+    if norm == 0:
+        return v
+    return v / norm
+
+
+def embed_one(model, tokenizer, text, normalize=True):
+    tokens = tokenizer(text, return_tensors="pt", truncation=True)
+    with torch.no_grad():
+        embedding = model.model.encoder(**tokens).last_hidden_state.mean(axis=1)
+        embedding = embedding.detach().numpy()[0]
+
+    if normalize:
+        return normalize_vector(embedding)
+    else:
+        return embedding
+
+
+def knn(query_embedding, embeddings, df, k=5, hebrew=True):
+    sims = np.dot(embeddings, query_embedding.T)
+    outs = np.argsort(sims, axis=0)[-k:][::-1]
+    select = outs.ravel()
+    if hebrew:
+        return df.iloc[select][["arabic", "hebrew", "validated"]]
+    else:
+        return df.iloc[select][["arabic", "english", "validated"]]
+
+def run_knn(text, k=5):
+    print(text)
+    query_embedding = embed_one(translate.model_from_ar,
+                                translate.tokenizer_from_ar, text)
+    return knn(query_embedding, embeddings, data, k=k, hebrew=True)
+
+
+def style_dataframe(df):
+    styled_df = df.style.set_properties(**{
+        'font-family': 'Arial, sans-serif',
+        'font-size': '20px',
+        'text-align': 'right',
+        'direction': 'rtl',
+        'align': 'right'
+    }).set_table_styles([
+    {'selector': 'th', 'props': [('text-align', 'right')]}
+])
+    return styled_df
+
+
+def style_dataframe(df):
+    return df.style.set_table_styles([
+        {'selector': 'thead', 'props': [('text-align', 'right')]},
+        {'selector': '.index_name', 'props': [('text-align', 'right')]},
+    ]).set_properties(**{
+        'text-align': 'right',
+    })  # Replace 'column_name' with your actual column name
+
+
+def update_df(hidden_arabic):
+    df = run_knn(hidden_arabic, 100)
+    # replace true and false in validated column with checkmark and x emoji
+    df["validated"] = df["validated"].apply(lambda x: "✅" if x else "❌")
+    # replace name validated with "מאומת"
+    df = df.rename(columns={"validated": "מאומת"})
+    # replace name arabic with "ערבית"
+    df = df.rename(columns={"arabic": "ערבית"})
+    # replace name hebrew with "עברית"
+    df = df.rename(columns={"hebrew": "עברית"})
+    styled_df = style_dataframe(df)
+    return gr.DataFrame(value=styled_df, visible=True)

translate.py

@@ -0,0 +1,101 @@
+import torch
+from transformers import MarianMTModel, AutoTokenizer
+import ctranslate2
+from colorize import align_words
+import logging
+
+# Create a logger
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)  # Set to debug to capture all levels of logs
+file_handler = logging.FileHandler('app.log', mode='a')  # 'a' mode appends to the file
+file_handler.setLevel(logging.INFO)
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+file_handler.setFormatter(formatter)
+logger.addHandler(file_handler)
+
+model_to_ar = MarianMTModel.from_pretrained("./he_ar/", output_attentions=True)
+model_from_ar = MarianMTModel.from_pretrained("./ar_he/", output_attentions=True)
+model_to_ar_ct2 = ctranslate2.Translator("./he_ar_ct2/")
+model_from_ar_ct2 = ctranslate2.Translator("./ar_he_ct2/")
+
+tokenizer_to_ar = AutoTokenizer.from_pretrained("./he_ar/")
+tokenizer_from_ar = AutoTokenizer.from_pretrained("./ar_he/")
+print("Done loading models")
+
+dialect_map = {
+    "Palestinian": "P",
+    "Syrian": "S",
+    "Lebanese": "L",
+    "Egyptian": "E",
+    "פלסטיני": "P",
+    "סורי": "S",
+    "לבנוני": "L",
+    "מצרי": "E"
+}
+
+
+def translate(text, ct_model, hf_model, tokenizer, to_arabic=True,
+              threshold=None, layer=2, head=6):
+
+    logger.info(f"Translating: {text}")
+    inp_tokens = tokenizer.convert_ids_to_tokens(tokenizer.encode(text))
+    out_tokens = ct_model.translate_batch([inp_tokens])[0].hypotheses[0]
+    out_string = tokenizer.convert_tokens_to_string(out_tokens)
+
+    encoder_input_ids = torch.tensor(tokenizer.convert_tokens_to_ids(inp_tokens)).unsqueeze(0)
+    decoder_input_ids = torch.tensor(tokenizer.convert_tokens_to_ids(["<pad>"] + out_tokens + 
+                                                                     ['</s>'])).unsqueeze(0)
+    
+    colorization_output = hf_model(input_ids=encoder_input_ids,
+                                   decoder_input_ids=decoder_input_ids)
+    
+    if not threshold:
+        if len(inp_tokens) < 10:
+            threshold = 0.05
+        elif len(inp_tokens) < 20:
+            threshold = 0.10
+        else:
+            threshold = 0.05
+
+    srchtml, tgthtml = align_words(colorization_output,
+                                   tokenizer,
+                                   encoder_input_ids,
+                                   decoder_input_ids,
+                                   threshold,
+                                   skip_first_src=to_arabic,
+                                   skip_second_src=False,
+                                   layer=layer,
+                                   head=head)
+
+    html = f"<div style='direction: rtl'>{srchtml}<br><br>{tgthtml}</div>"
+    
+    arabic = out_string if is_arabic(out_string) else text
+    return html, arabic
+
+
+#%%
+
+
+def is_arabic(text):
+    # return True if text has more than 50% arabic characters, False otherwise
+    text = text.replace(" ", "")
+    arabic_chars = 0
+    for c in text:
+        if "\u0600" <= c <= "\u06FF":
+            arabic_chars += 1
+            
+    return arabic_chars / len(text) > 0.5
+
+def run_translate(text, dialect=None):
+    if not text:
+        return ""
+    if is_arabic(text):
+        return translate(text, model_from_ar_ct2, model_from_ar, tokenizer_from_ar,
+                         to_arabic=False, threshold=None, layer=2, head=1)
+    else:
+        if dialect in dialect_map:
+            dialect = dialect_map[dialect]
+
+        text = f"{dialect} {text}" if dialect else text
+        return translate(text, model_to_ar_ct2, model_to_ar, tokenizer_to_ar,
+                          to_arabic=True, threshold=None, layer=2, head=6)

translit.py

@@ -0,0 +1,158 @@
+#%%
+from transformers import CanineForTokenClassification, AutoTokenizer
+import re
+import torch
+
+# instantiate module logger
+import logging
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+model = CanineForTokenClassification.from_pretrained("guymorlan/levanti_diacritics2translit")
+tokenizer = AutoTokenizer.from_pretrained("guymorlan/levanti_diacritics2translit")
+
+#%%
+
+def diacritics2hebrew_vowels(text, model, tokenizer):
+    tokens = tokenizer(text, return_tensors="pt")
+    with torch.no_grad():
+        pred = model(**tokens)
+        pred = pred.logits.argmax(-1).tolist()
+        
+    pred = pred[0][1:-1] # remove CLS and SEP
+    output = []
+    for p, c in zip(pred, text):
+        if p != model.config.label2id["O"]:
+            output.append(model.config.id2label[p])
+        else:
+            output.append(c)
+    output = "".join(output)
+    
+    # print("Done converting to Hebrew vowels")
+    logger.warning("Done converting to Hebrew vowels")
+    return output
+
+#%%
+
+
+arabic_to_english = {
+    "ا": "a", "أ": "a", "إ": "a", "ء": "a", "ئ": "a", "ؤ": "a",
+    "آ": "aa", "ى": "a", "ب": "b", "ت": "t", "ث": "th", "ج": "j",
+    "ح": "h", "خ": "kh", "د": "d", "ذ": "dh", "ر": "r", "ز": "z",
+    "س": "s", "ش": "sh", "ص": "s", "ض": "d", "ط": "t", "ظ": "z",
+    "ع": "a", "غ": "gh", "ف": "f", "ق": "q", "ك": "k", "ل": "l",
+    "م": "m", "ن": "n", "ه": "h", "و": "w", "ي": "y", "ة": "h",
+    "َ": "a", "ُ": "u", "ِ": "i",
+    "،": ",",
+    "ֹ": "o",  # holam
+    "ַ": "a",  # patah
+    "ִ": "i",  # hiriq
+    "ְ": "",   # shva
+    "ֻ": "u",  # kubutz
+    'ֵ': "e",
+    "ّ": "SHADDA"  # shadda
+}
+
+arabic_to_hebrew = {
+    # regular letters
+    "ا": "א", "أ": "א", "إ": "א", "ء": "א", "ئ": "א", "ؤ": "א", 
+    "آ": "אא", "ى": "א", "ب": "בּ", "ت": "ת", "ث": "ת'", "ج": "ג'", 
+    "ح": "ח", "خ": "ח'", "د": "ד", "ذ": "ד'", "ر": "ר", "ز": "ז", 
+    "س": "ס", "ش": "ש", "ص": "צ", "ض": "צ'", "ط": "ט", "ظ": "ט'", 
+    "ع": "ע", "غ": "ע'", "ف": "פ", "ق": "ק", "ك": "כּ", "ل": "ל", 
+    "م": "מ", "ن": "נ", "ه": "ה", "و": "ו", "ي": "י", "ة": "ה",
+    # special characters
+    "،": ",", "َ": "ַ", "ُ": "ֻ", "ِ": "ִ",
+    "؟": "?", "؛": ";", "ـ": "",
+    # shadda to \u0598
+    "ّ": "\u0598",
+}
+
+vowels = ["،", ",", "َ", "ַ", "ُ", "ֻ", "ِ", "ִ", 'ֵ']
+
+final_letters = {
+    "ن": "ן", "م": "ם", "ص": "ץ", "ض": "ץ'", "ف": "ף",
+}
+
+def reorder_hebrew_nikkud(input_string):
+    # in case of 2 character letter transliteration, move the nikkud back to the first character and not the apostrophe
+
+    # Define a dictionary for the nikkud signs
+    nikkud_signs = {
+        "ֹ": "o",  # holam
+        "ַ": "a",  # patah
+        "ִ": "i",  # hiriq
+        "ְ": "",   # shva
+        "ֻ": "u",  # kubutz
+        "ֵ": "e",  # tsere
+        "ّ": "SHADDA"  # shadda
+    }
+    
+    # Create a regex pattern for Hebrew letter followed by apostrophe and one of the nikkud signs
+    pattern = r'([\u0590-\u05FF])(\')([' + ''.join(nikkud_signs.keys()) + '])'
+    replacement = r'\1\3\2'
+    
+    result = re.sub(pattern, replacement, input_string)
+    
+    return result
+
+def reverse_holam_shadda_vav(input_string):
+    # For better readability, replace (holam, shadda, ו) with (shadda, ו, holam)
+    # instead of shadda we use the replacement \u0598
+    pattern = r'(\u05B9)(\u0598)(\u05D5)'
+    replacement = r'\2\3\1'
+    result = re.sub(pattern, replacement, input_string)
+    
+    return result
+
+def to_taatik(arabic):
+    taatik = []
+    for index, letter in enumerate(arabic):
+        if (
+            (index == len(arabic) - 1 or arabic[index + 1] in {" ", ".", "،"}) and 
+            letter in final_letters
+        ):
+            taatik.append(final_letters[letter])
+        elif letter not in arabic_to_hebrew:
+            taatik.append(letter)
+        else:
+            taatik.append(arabic_to_hebrew[letter])
+    reversed = reverse_holam_shadda_vav("".join(taatik))
+    reordered = reorder_hebrew_nikkud(reversed)
+    # print("Done converting to taatik")
+    logger.warning("Done converting to taatik")
+    return reordered
+
+
+
+
+def to_translit(arabic):
+    translit = []
+    for letter in arabic:
+        if letter not in arabic_to_english:
+            translit.append([letter, letter])
+        else:
+            if arabic_to_english[letter] == "SHADDA":
+                if translit[-1][0] in vowels:
+                    translit[-2][1] = translit[-2][1].upper()
+                else:
+                    translit[-1][1] = translit[-1][1].upper()
+    
+            else:
+                translit.append([letter, arabic_to_english[letter]])
+            
+    return "".join([x[1] for x in translit])
+
+
+# %%
+
+def taatik(text):
+    return to_taatik(diacritics2hebrew_vowels(text, model, tokenizer))
+
+# text = "لَازِم نِعْطِي رَشَّات وِقَائِيِّة لِل��َّجَر "
+# heb_vowels = diacritics2hebrew_vowels(text, model, tokenizer)
+# #%%
+# to_taatik(heb_vowels)
+# #%%
+# to_translit(heb_vowels)
+# # %%

tts.py

@@ -0,0 +1,82 @@
+#%%
+import azure.cognitiveservices.speech as speechsdk
+import re
+import os
+import hashlib
+import random
+
+speech_config = speechsdk.SpeechConfig(subscription=os.environ.get('SPEECH_KEY'),
+                                       region=os.environ.get('SPEECH_REGION'))
+
+def do_cleanup(dir='wavs', num_files=100):
+    files = os.listdir(dir)
+    if len(files) > num_files:
+        for file in files[:len(files) - num_files]:
+            os.remove(f"{dir}/{file}")
+
+def add_sukun(text):
+    # Define Arabic letters and sukun
+    arabic_letters = 'اأإآةبتثجحخدذرزسشصضطظعغفقكلمنهوي'
+    shadda = 'ّ'
+    arabic_letters += shadda
+    sukun = 'ْ'
+    punctuation = '.,;!?،؛؟'
+
+    def process_word(word):
+        # If the last character is punctuation, process the letter before it
+        if word[-1] in punctuation:
+            if len(word) > 1 and word[-2] in arabic_letters and word[-2] != sukun:
+                return word[:-2] + word[-2] + sukun + word[-1]
+            return word
+        # If the last character is an Arabic letter and does not have a sukun, add one
+        elif word[-1] in arabic_letters and word[-1] != sukun:
+            return word + sukun
+        return word
+
+    # Use regex to split text into words and punctuation
+    words = re.findall(r'\S+|[.,;!?،؛؟]', text)
+    processed_text = ' '.join(process_word(word) for word in words)
+    return processed_text
+
+def get_ssml(text, voice='de-DE-SeraphinaMultilingualNeural'):
+    return f'<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xml:lang="ar-SA"><voice name="{voice}"><lang xml:lang="ar-SA">{text}</lang></voice></speak>'
+
+
+def get_audio(input_text, voice='de-DE-FlorianMultilingualNeural', use_ssml=True):
+
+    input_text = add_sukun(input_text)
+    hash = hashlib.md5(input_text.encode()).hexdigest()
+    
+    if os.path.exists(f"wavs/{hash}.wav"):
+        return f"wavs/{hash}.wav"
+    
+    audio_config = speechsdk.audio.AudioOutputConfig(filename=f"wavs/{hash}.wav")
+    # speech_config.speech_synthesis_voice_name=voice
+    # speech_config.speech_synthesis_language = "ar-EG"
+    speech_config.set_speech_synthesis_output_format(
+        speechsdk.SpeechSynthesisOutputFormat.Riff24Khz16BitMonoPcm
+    )
+
+    speech_synthesizer = speechsdk.SpeechSynthesizer(speech_config=speech_config,
+                                                     audio_config=audio_config)
+    if use_ssml:
+        # print("Using SSML")
+        ssml = get_ssml(input_text, voice=voice)
+        result = speech_synthesizer.speak_ssml_async(ssml).get()
+    else:
+        # print("Using text")
+        result = speech_synthesizer.speak_text_async(input_text).get()
+    
+    if result.reason == speechsdk.ResultReason.SynthesizingAudioCompleted:
+        print("Speech synthesized for text [{}]".format(input_text))
+    elif result.reason == speechsdk.ResultReason.Canceled:
+        cancellation_details = result.cancellation_details
+        print("Speech synthesis canceled: {}".format(cancellation_details.reason))
+        if cancellation_details.reason == speechsdk.CancellationReason.Error:
+            print("Error details: {}".format(cancellation_details.error_details))
+
+    # randomly every 50 calls, clean up the wavs folder
+    if random.randint(1, 50) == 1:
+        do_cleanup()
+
+    return f"wavs/{hash}.wav"