katuni4ka commited on
Commit
1470032
1 Parent(s): 0fc7408

Update tokenization_chatglm.py

Browse files
Files changed (1) hide show
  1. tokenization_chatglm.py +3 -3
tokenization_chatglm.py CHANGED
@@ -70,9 +70,9 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
70
 
71
  self.vocab_file = vocab_file
72
  self.tokenizer = SPTokenizer(vocab_file)
73
- kwargs.pop("eos_token")
74
- kwargs.pop("pad_token")
75
- kwargs.pop("unk_token")
76
  self.special_tokens = {
77
  "<bos>": self.tokenizer.bos_id,
78
  "<eos>": self.tokenizer.eos_id,
 
70
 
71
  self.vocab_file = vocab_file
72
  self.tokenizer = SPTokenizer(vocab_file)
73
+ kwargs.pop("eos_token", None)
74
+ kwargs.pop("pad_token", None)
75
+ kwargs.pop("unk_token", None)
76
  self.special_tokens = {
77
  "<bos>": self.tokenizer.bos_id,
78
  "<eos>": self.tokenizer.eos_id,