if001 commited on
Commit
88b31c9
1 Parent(s): 2faf61e
Files changed (1) hide show
  1. sentencepiece_ja.py +4 -0
sentencepiece_ja.py CHANGED
@@ -46,6 +46,10 @@ class SentencePieceJA(PreTrainedTokenizer):
46
  def _convert_id_to_token(self, index: int) -> str:
47
  return self._tokenizer.decode([index])
48
 
 
 
 
 
49
  def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
50
  index = 0
51
  if os.path.isdir(save_directory):
 
46
  def _convert_id_to_token(self, index: int) -> str:
47
  return self._tokenizer.decode([index])
48
 
49
+ def convert_tokens_to_string(self, tokens: List[str]) -> str:
50
+ ## 日本語用
51
+ return "".join(tokens)
52
+
53
  def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
54
  index = 0
55
  if os.path.isdir(save_directory):