Add TikToken extention support for the Hebrew Tokenizer

tiktoken/tests/test_compare_hebrew.py

@@ -0,0 +1,23 @@
+import tiktoken
+
+test_string = "האיש האחרון עלי אדמות ישב לבד בחדרו, כשלפתע נשמעה דפיקה בדלת"
+
+print(f'Test string = "{test_string}"')
+
+enc = tiktoken.get_encoding("cl100k_base")
+encoded_text = enc.encode(test_string)
+print(f'num of characters = {len(test_string)} encoded length = {len(encoded_text)} (cl100k_base)')
+decoded_text = enc.decode(encoded_text)
+assert  decoded_text == test_string
+
+enc = tiktoken.get_encoding("gpt2")
+encoded_text = enc.encode(test_string)
+print(f'num of characters = {len(test_string)} encoded length = {len(encoded_text)} (gpt2)')
+decoded_text = enc.decode(encoded_text)
+assert  decoded_text == test_string
+
+enc = tiktoken.get_encoding("gpt-hebrew-tokenizer")
+encoded_text = enc.encode(test_string)
+print(f'num of characters = {len(test_string)} encoded length = {len(encoded_text)} (gpt-hebrew-tokenizer)')
+decoded_text = enc.decode(encoded_text)
+assert  decoded_text == test_string

tiktoken/tiktoken_ext/tiktoken_ext_norod78_hf.py

@@ -0,0 +1,32 @@
+from tiktoken.load import data_gym_to_mergeable_bpe_ranks, load_tiktoken_bpe
+
+def gpt_j_hebrew_tokenizer():
+    mergeable_ranks = data_gym_to_mergeable_bpe_ranks(
+        vocab_bpe_file="https://huggingface.co/Norod78/gpt-j-hebrew-tokenizer/raw/main/merges.txt",
+        encoder_json_file="https://huggingface.co/Norod78/gpt-j-hebrew-tokenizer/raw/main/vocab.json",
+    )
+    return {
+        "name": "gpt-j-hebrew-tokenizer",
+        "explicit_n_vocab": 50257,
+        "pat_str": r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""",
+        "mergeable_ranks": mergeable_ranks,
+        "special_tokens": {"<|endoftext|>": 50256},
+    }
+
+def gpt_hebrew_tokenizer():
+    mergeable_ranks = data_gym_to_mergeable_bpe_ranks(
+        vocab_bpe_file="https://huggingface.co/Norod78/TinyStories-3M-val-Hebrew/raw/main/merges.txt",
+        encoder_json_file="https://huggingface.co/Norod78/TinyStories-3M-val-Hebrew/raw/main/vocab.json",
+    )
+    return {
+        "name": "gpt-hebrew-tokenizer",
+        "explicit_n_vocab": 50259,
+        "pat_str": r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""",
+        "mergeable_ranks": mergeable_ranks,
+        "special_tokens": {"<|endoftext|>": 50256, "<|startoftext|>": 50257, "<|pad|>": 50258},
+    }
+
+ENCODING_CONSTRUCTORS = {
+    "gpt-j-hebrew-tokenizer": gpt_j_hebrew_tokenizer,
+    "gpt-hebrew-tokenizer": gpt_hebrew_tokenizer,
+    }