if001 commited on
Commit
0e4910a
1 Parent(s): 15608e1

fix vocab_size to property

Browse files
Files changed (2) hide show
  1. sentencepiece_ja.py +3 -2
  2. tokenizer_config.json +2 -1
sentencepiece_ja.py CHANGED
@@ -28,10 +28,11 @@ class SentencePieceJA(PreTrainedTokenizer):
28
  'unk_token': unk,
29
  'mask_token': mask
30
  })
31
-
32
  def get_vocab(self) -> int:
33
  return self._tokenizer.get_vocab()
34
-
 
35
  def vocab_size(self) -> int:
36
  return self._tokenizer.get_vocab_size()
37
 
 
28
  'unk_token': unk,
29
  'mask_token': mask
30
  })
31
+
32
  def get_vocab(self) -> int:
33
  return self._tokenizer.get_vocab()
34
+
35
+ @property
36
  def vocab_size(self) -> int:
37
  return self._tokenizer.get_vocab_size()
38
 
tokenizer_config.json CHANGED
@@ -9,5 +9,6 @@
9
  "tokenizer_class": "SentencePieceJA",
10
  "auto_map": {
11
  "AutoTokenizer": ["","sentencepiece_ja.SentencePieceJA"]
12
- }
 
13
  }
 
9
  "tokenizer_class": "SentencePieceJA",
10
  "auto_map": {
11
  "AutoTokenizer": ["","sentencepiece_ja.SentencePieceJA"]
12
+ },
13
+ "transformers_version": " 4.34.1"
14
  }