Update README.md

README.md

@@ -23,7 +23,8 @@ import os
 os.environ['VLLM_ATTENTION_BACKEND'] = 'FLASHINFER'
 
 from transformers import AutoTokenizer
-from vllm import LLM, SamplingParams
+from gptqmodel import BACKEND, GPTQModel
+from vllm import SamplingParams
 
 model_name = "ModelCloud/gemma-2-27b-it-gptq-4bit"
 
@@ -31,13 +32,20 @@ prompt = [{"role": "user", "content": "I am in Shanghai, preparing to visit the
 
 tokenizer = AutoTokenizer.from_pretrained(model_name)
 
-llm = LLM(
-    model=model_name,
-)
+model = GPTQModel.from_quantized(
+            model_name,
+            backend=BACKEND.VLLM,
+        )
+
 sampling_params = SamplingParams(temperature=0.95, max_tokens=128)
 
 inputs = tokenizer.apply_chat_template(prompt, tokenize=False, add_generation_prompt=True)
-outputs = llm.generate(prompts=inputs, sampling_params=sampling_params)
+
+outputs = model.generate(
+            prompts=inputs,
+            sampling_params=sampling_params,
+        )
 
 print(outputs[0].outputs[0].text)
+
 ```