Upload 5 files

README.md

@@ -1,3 +1,17 @@
 ---
 license: apache-2.0
 ---
+
+### How to use
+### 
+```
+from transformers import LlamaTokenizerFast
+tokenizer = LlamaTokenizerFast.from_pretrained("mimir-project/tokenizer", token=True)
+```
+or
+```
+from transformers import AutoTokenizer
+tokenizer = AutoTokenizer.from_pretrained("mimir-project/tokenizer", token=True)
+```
+
+Copied from https://github.com/SmartmediaAI/MIMIR-project/tree/main

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

tokenizer_config.json

@@ -0,0 +1,51 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "bos_token_id": 1,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "eos_token_id": 2,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "pad_token_id": 3,
+  "padding_side": "right",
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "unk_token_id": 0,
+  "use_default_system_prompt": false
+}

tokenizer_train.py

@@ -0,0 +1,140 @@
+import json
+import argparse
+from tqdm import tqdm
+import os
+
+from datasets import load_dataset
+from tokenizers import SentencePieceBPETokenizer
+from transformers import LlamaTokenizerFast, TrainingArguments, AutoTokenizer
+
+def main(args):
+
+	# Load the dataset from the huggingface Hub and prepare it for training
+	if args.dataset_name is not None:
+		data_files = os.listdir(args.dataset_name)
+		data_files = [args.dataset_name+f for f in data_files]
+		print(len(data_files))
+		dataset = load_dataset("json",
+				data_files=data_files,
+				split=args.dataset_split,
+				token=args.hub_token if args.hub_token else None
+		)
+		print(dataset)
+
+	else:
+		raise ValueError("No dataset name provided or dataset is already tokenized") 
+
+	# Remove non text columns
+	dataset = dataset.remove_columns([col for col in dataset.column_names if col != "text"])
+
+	# select `num_samples` from the dataset
+	dataset = dataset.shuffle(seed=args.seed).select(range(args.num_samples))
+
+	# Create a SentencePieceBPETokenizer
+	tokenizer = SentencePieceBPETokenizer()
+
+	# Train the SentencePieceBPETokenizer on the dataset
+	tokenizer.train_from_iterator(
+		iterator=dataset['text'],
+		vocab_size=args.vocab_size,
+		show_progress=True,
+		special_tokens=["<unk>", "<s>", "</s>",  "<pad>"],
+	)
+
+	# Save the tokenizer
+	tokenizer.save("new-sentencepiece-tokenizer.json", pretty=True)
+
+	# Load reference tokenizer
+	if args.reference_tokenizer is not None and args.hub_token is not None:
+		reference_tokenizer = AutoTokenizer.from_pretrained(args.reference_tokenizer, token=args.hub_token if args.hub_token else None)
+		reference_tokenizer.save_pretrained("reference-tokenizer")
+	else:
+		raise ValueError("No tokenizer name provided or no hub token provided. Try using `--reference_tokenizer 'meta-llama/Llama-2-7b-hf'")
+
+	# Read and dump the json file for the new tokenizer and the reference tokenizer
+	with open("new-sentencepiece-tokenizer.json") as f:
+		new_llama_tokenizer_json = json.load(f)
+
+	with open("reference-tokenizer/tokenizer.json") as f:
+		reference_tokenizer_json = json.load(f)
+
+	# Add the reference tokenizer's config to the new tokenizer's config
+	new_llama_tokenizer_json["normalizer"] = reference_tokenizer_json["normalizer"]
+	new_llama_tokenizer_json["pre_tokenizer"] = reference_tokenizer_json["pre_tokenizer"]
+	new_llama_tokenizer_json["post_processor"] = reference_tokenizer_json["post_processor"]
+	new_llama_tokenizer_json["decoder"] = reference_tokenizer_json["decoder"]
+	new_llama_tokenizer_json["model"]['fuse_unk'] = reference_tokenizer_json["model"]['fuse_unk']
+	new_llama_tokenizer_json["model"]['byte_fallback'] = reference_tokenizer_json["model"]['byte_fallback']
+
+	# Dump the new tokenizer's config
+	with open("new-sentencepiece-tokenizer.json", "w") as f:
+		json.dump(new_llama_tokenizer_json, f, indent=2, ensure_ascii=False)
+
+	# Load the new tokenizer as a LlamaTokenizerFast
+	new_llama_tokenizer = LlamaTokenizerFast(
+		tokenizer_file="new-sentencepiece-tokenizer.json",
+		name_or_path=args.reference_tokenizer + "-tokenizer",
+		unk_token="<unk>",
+		unk_token_id=0,
+		bos_token="<s>",
+		bos_token_id=1,
+		eos_token="</s>",
+		eos_token_id=2,
+		pad_token="<pad>",
+		pad_token_id=3,
+		padding_side="right",
+	)
+
+	# Save the new tokenizer
+	new_llama_tokenizer.save_pretrained("new-llama-tokenizer")
+
+if __name__ == "__main__":
+	parser = argparse.ArgumentParser(description="Train a new Llama tokenizer")
+	parser.add_argument(
+		"--dataset_name",
+		type=str,
+		default=None,
+		help="The name of the dataset to be tokenized",
+	)
+	parser.add_argument(
+		"--dataset_split",
+		type=str,
+		default=None,
+		help="The split of the dataset to be tokenized",
+	)
+	parser.add_argument(
+		"--hub_token",
+		type=str,
+		default=None,
+		help="The token to access the dataset on the hub",
+	)
+	parser.add_argument(
+		"--reference_tokenizer",
+		type=str,
+		default=None,
+		help="The name of the reference tokenizer to use",
+	)
+	parser.add_argument(
+		"--seed",
+		type=int,
+		default=123,
+		help="set random seed",
+	)
+	parser.add_argument(
+		"--num_samples",
+		type=int,
+		default=None,
+		help="Number of samples to use from the dataset",
+	)
+	parser.add_argument(
+		"--vocab_size",
+		type=int,
+		default=None,
+		help="Vocabulary size to use for the tokenizer",
+	)
+	args = parser.parse_args()
+	main(args)
+
+# How to run:
+# python tokenizer_train.py --dataset_name /mimir/dataset/delivery/mimir_base/data/ --dataset_split train --reference_tokenizer meta-llama/Llama-2-7b-hf --vocab_size 32768 --hub_token hf_IIbKlx.... --num_samples 6000000
+