Fixed AutoGPTQ example code

README.md

@@ -189,30 +189,25 @@ Then try the following example code:
 ```python
 from transformers import AutoTokenizer, pipeline, logging
 from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig
+# model_name_or_path = "RuterNorway/Llama-2-13b-chat-norwegian-GPTQ"
 model_name_or_path = "RuterNorway/Llama-2-13b-chat-norwegian-GPTQ"
 model_basename = "gptq_model-4bit-128g"
 use_triton = False
 tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=True)
+quantize_config = None
 model = AutoGPTQForCausalLM.from_quantized(model_name_or_path,
-        model_basename=model_basename
-        use_safetensors=True,
-        trust_remote_code=True,
-        device="cuda:0",
-        use_triton=use_triton,
-        quantize_config=None)
-"""
-To download from a specific branch, use the revision parameter, as in this example:
-model = AutoGPTQForCausalLM.from_quantized(model_name_or_path,
-        revision="gptq-4bit-32g-actorder_True",
         model_basename=model_basename,
         use_safetensors=True,
         trust_remote_code=True,
         device="cuda:0",
-        quantize_config=None)
-"""
-prompt = "Fortell meg om AI"
-prompt_template=f'''### Human: {prompt}
-### Assistant:
+        use_triton=use_triton,
+        quantize_config=quantize_config)
+
+instruction = "Gi en vurdering (positiv/negativ) og 4 stikkord som forklarer vurderingen. Svar i dette formatet: vurdering: positiv/negativ \n,stikkord: \n"
+input = "Bussjåføren på Snarøya 31 (12.26 bussen på Årvoll senter) som var på vei ut av holdeplassen men venta da han så jeg løp til bussen og ikke var langt unna. You made my day!"
+prompt_template=f'''### Instruction: {instruction}
+### Input: {input}
+### Response:
 '''
 print("\n\n*** Generate:")
 input_ids = tokenizer(prompt_template, return_tensors='pt').input_ids.cuda()
@@ -221,10 +216,11 @@ print(tokenizer.decode(output[0]))
 # Inference can also be done using transformers' pipeline
 # Prevent printing spurious transformers error when using pipeline with AutoGPTQ
 logging.set_verbosity(logging.CRITICAL)
-print("*** Pipeline:")
+print("\n\n*** Pipeline:\n\n")
 pipe = pipeline(
     "text-generation",
     model=model,
+    do_sample=True,
     tokenizer=tokenizer,
     max_new_tokens=512,
     temperature=0.7,