katuni4ka commited on
Commit
0fc7408
1 Parent(s): 3418503

Update tokenization_chatglm.py

Browse files
Files changed (1) hide show
  1. tokenization_chatglm.py +1 -1
tokenization_chatglm.py CHANGED
@@ -73,12 +73,12 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
73
  kwargs.pop("eos_token")
74
  kwargs.pop("pad_token")
75
  kwargs.pop("unk_token")
76
- super().__init__(padding_side=padding_side, clean_up_tokenization_spaces=clean_up_tokenization_spaces, **kwargs)
77
  self.special_tokens = {
78
  "<bos>": self.tokenizer.bos_id,
79
  "<eos>": self.tokenizer.eos_id,
80
  "<pad>": self.tokenizer.pad_id
81
  }
 
82
 
83
  def get_command(self, token):
84
  if token in self.special_tokens:
 
73
  kwargs.pop("eos_token")
74
  kwargs.pop("pad_token")
75
  kwargs.pop("unk_token")
 
76
  self.special_tokens = {
77
  "<bos>": self.tokenizer.bos_id,
78
  "<eos>": self.tokenizer.eos_id,
79
  "<pad>": self.tokenizer.pad_id
80
  }
81
+ super().__init__(padding_side=padding_side, clean_up_tokenization_spaces=clean_up_tokenization_spaces, **kwargs)
82
 
83
  def get_command(self, token):
84
  if token in self.special_tokens: