fix
Browse files- sentencepiece_ja.py +2 -2
sentencepiece_ja.py
CHANGED
@@ -41,8 +41,8 @@ class SentencePieceJA(PreTrainedTokenizer):
|
|
41 |
def _convert_token_to_id(self, token):
|
42 |
return self._tokenizer.encode(token).ids[0]
|
43 |
|
44 |
-
def _convert_id_to_token(self, index: int) -> str:
|
45 |
-
return self._tokenizer.decode(index)
|
46 |
|
47 |
def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
|
48 |
index = 0
|
|
|
41 |
def _convert_token_to_id(self, token):
|
42 |
return self._tokenizer.encode(token).ids[0]
|
43 |
|
44 |
+
def _convert_id_to_token(self, index: int) -> str:
|
45 |
+
return self._tokenizer.decode([index])
|
46 |
|
47 |
def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
|
48 |
index = 0
|