load from cache

sentencepiece_ja.py

@@ -1,8 +1,8 @@
 import os 
 from typing import Union, List, Optional, Tuple
 
-from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
-
+from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast, AutoTokenizer
+from transformers.utils.hub import cached_file
 class SentencePieceJA(PreTrainedTokenizer):
     def __init__(self, 
                  model_path = "./tokenizer.json", 
@@ -11,9 +11,15 @@ class SentencePieceJA(PreTrainedTokenizer):
                  eos = "<EOS>",
                  unk = "<UNK>",
                  mask = "<MASK>",
-                 **kwargs):        
+                 **kwargs):
         from tokenizers import Tokenizer
-        self._tokenizer = Tokenizer.from_file(model_path)
+        try:             
+            self._tokenizer = Tokenizer.from_file(model_path)
+        except Exception as e:
+            print('exception: ', e)
+            print('load from cache...')
+            model_path = cached_file('if001/sentencepiece_ja', 'tokenizer.json')        
+            self._tokenizer = Tokenizer.from_file(model_path)
         super().__init__(**kwargs)
         self.add_special_tokens({
             'pad_token': pad,