TMElyralab

/

lyraChatGLM

like 107

Follow

Tencent Music Entertainment Lyra Lab 43

Traceback (most recent call last):
File "/usr/lyraChatGLM/demo.py", line 10, in
model = LyraChatGLM6B(model_path, tokenizer_path, data_type, int8_mode, arch)
File "/usr/lyraChatGLM/lyraChatGLM/lyra_glm.py", line 24, in init
self.model, self.tokenizer = self.load_model_and_tokenizer()
File "/usr/lyraChatGLM/lyraChatGLM/lyra_glm.py", line 37, in load_model_and_tokenizer
tokenizer = transformers.AutoTokenizer.from_pretrained(tokenizer_path, trust_remote_code=True)
File "/root/miniconda3/envs/ai/lib/python3.10/site-packages/transformers/models/auto/tokenization_auto.py", line 678, in from_pretrained
return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
File "/root/miniconda3/envs/ai/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 1825, in from_pretrained
return cls._from_pretrained(
File "/root/miniconda3/envs/ai/lib/python3.10/site-packages/transformers/tokenization_utils_base.py", line 1988, in _from_pretrained
tokenizer = cls(*init_inputs, **init_kwargs)
File "/root/.cache/huggingface/modules/transformers_modules/models/tokenization_chatglm.py", line 221, in init
self.sp_tokenizer = SPTokenizer(vocab_file, num_image_tokens=num_image_tokens)
File "/root/.cache/huggingface/modules/transformers_modules/models/tokenization_chatglm.py", line 64, in init
self.text_tokenizer = TextTokenizer(vocab_file)
File "/root/.cache/huggingface/modules/transformers_modules/models/tokenization_chatglm.py", line 22, in init
self.sp.Load(model_path)
File "/root/miniconda3/envs/ai/lib/python3.10/site-packages/sentencepiece/init.py", line 905, in Load
return self.LoadFromFile(model_file)
File "/root/miniconda3/envs/ai/lib/python3.10/site-packages/sentencepiece/init.py", line 310, in LoadFromFile
return _sentencepiece.SentencePieceProcessor_LoadFromFile(self, arg)
RuntimeError: Internal: src/sentencepiece_processor.cc(1101) [model_proto->ParseFromArray(serialized.data(), serialized.size())]

怎么解决呢，谢谢