neuralmagic
/

DeepSeek-Coder-V2-Instruct-FP8

Text Generation

text-generation-inference

Inference Endpoints

Model card Files Files and versions Community

Lin-K76 commited on Jul 22

Commit

7014671

•

1 Parent(s): c7fb3ef

Update README.md

Files changed (1) hide show

README.md +1 -1

README.md CHANGED Viewed

@@ -56,7 +56,7 @@ messages = [
     {"role": "user", "content": "Who are you?"},
 ]
-prompts = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
 llm = LLM(model=model_id, trust_remote_code=True, max_model_len=4096)

     {"role": "user", "content": "Who are you?"},
 ]
+prompts = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, tensor_parallel_size=4)
 llm = LLM(model=model_id, trust_remote_code=True, max_model_len=4096)