Update messagers/message_composer.py

messagers/message_composer.py

@@ -151,7 +151,7 @@ class MessageComposer:
         # https://huggingface.co/NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO#prompt-format
         # https://huggingface.co/openchat/openchat-3.5-0106
         # https://huggingface.co/01-ai/Yi-1.5-34B-Chat
-        elif self.model in ["openchat-3.5", "command-r-plus", "gemma-7b", "yi-1.5-34b", "llama3-8b", "llama3-70b"]:
+        elif self.model in ["openchat-3.5", "command-r-plus", "gemma-7b", "yi-1.5-34b", "llama3-8b", "llama3-70b", "Qwen2-1.5B-Instruct", "Qwen2-7B-Instruct"]:
             # https://discuss.huggingface.co/t/error-with-new-tokenizers-urgent/2847/5
             tokenizer = AutoTokenizer.from_pretrained(
                 self.model_fullname, use_fast=False