if001 commited on
Commit
15608e1
1 Parent(s): bbd068b
Files changed (1) hide show
  1. sentencepiece_ja.py +2 -2
sentencepiece_ja.py CHANGED
@@ -41,8 +41,8 @@ class SentencePieceJA(PreTrainedTokenizer):
41
  def _convert_token_to_id(self, token):
42
  return self._tokenizer.encode(token).ids[0]
43
 
44
- def _convert_id_to_token(self, index: int) -> str:
45
- return self._tokenizer.decode(index)
46
 
47
  def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
48
  index = 0
 
41
  def _convert_token_to_id(self, token):
42
  return self._tokenizer.encode(token).ids[0]
43
 
44
+ def _convert_id_to_token(self, index: int) -> str:
45
+ return self._tokenizer.decode([index])
46
 
47
  def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
48
  index = 0