Update app.py

app.py

@@ -12,6 +12,10 @@ qwen_tokenizer = AutoTokenizer.from_pretrained('Qwen/Qwen2.5-0.5B-Instruct')
 ruadapt_tokenizer = AutoTokenizer.from_pretrained('msu-rcc-lair/RuadaptQwen2.5-32B-instruct')
 aya_tokenizer = AutoTokenizer.from_pretrained('mlx-community/aya-expanse-32b-8bit')
 claude_tokenizer = GPT2TokenizerFast.from_pretrained('Xenova/claude-tokenizer')
+xlmv_tokenizer = AutoTokenizer.from_pretrained('facebook/xlm-v-base')
+nllb_tokenizer = AutoTokenizer.from_pretrained('facebook/nllb-200-distilled-600M')
+
+
 
 ### User Interface ###
 st.title("Tokenization")
@@ -34,6 +38,8 @@ tokenizer = st.selectbox(
         "Aya-Expanse Tokenizer",
         "Open AI GPT-4o Tokenizer",
         "Anthropic Claude Tokenizer",
+        "XLM-V Tokenizer",
+        "NLLB-200 Tokenizer",
     ),
     index=None,
     placeholder="Select a tokenizer",
@@ -166,4 +172,25 @@ elif tokenizer == "Anthropic Claude Tokenizer":
         color = itertools.cycle(colors)
         st.write(stream_wp_token_ids)
 
+elif tokenizer == "XLM-V Tokenizer":
+    with st.expander("About XLM-V Tokenizer"):
+        st.write('')
+    ids = xlmv_tokenizer.encode(txt)
+    split_tokens = [xlmv_tokenizer.decode([t]) for t in ids]
+    st.write(stream_data)
+    if token_id == True:
+        color = itertools.cycle(colors)
+        st.write(stream_wp_token_ids)
+
+
+elif tokenizer == "NLLB-200 Tokenizer":
+    with st.expander("About NLLB-200 Tokenizer"):
+        st.write('')
+    ids = nllb_tokenizer.encode(txt)
+    split_tokens = [nllb_tokenizer.decode([t]) for t in ids]
+    st.write(stream_data)
+    if token_id == True:
+        color = itertools.cycle(colors)
+        st.write(stream_wp_token_ids)
+
 st.write(num_tokens(txt))