neuralmagic
/

DeepSeek-Coder-V2-Instruct-FP8

Text Generation

text-generation-inference

Inference Endpoints

Model card Files Files and versions Community

Lin-K76 commited on Jul 22

Commit

85dc941

•

1 Parent(s): 7014671

Update README.md

Files changed (1) hide show

README.md +2 -2

README.md CHANGED Viewed

@@ -56,9 +56,9 @@ messages = [
     {"role": "user", "content": "Who are you?"},
 ]
-prompts = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, tensor_parallel_size=4)
-llm = LLM(model=model_id, trust_remote_code=True, max_model_len=4096)
 outputs = llm.generate(prompts, sampling_params)

     {"role": "user", "content": "Who are you?"},
 ]
+prompts = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+llm = LLM(model=model_id, trust_remote_code=True, max_model_len=4096, tensor_parallel_size=4)
 outputs = llm.generate(prompts, sampling_params)