update the example for "Inference with Hugging Face library"

README.md

@@ -101,37 +101,47 @@ analysis is needed to assess potential harm or bias in the proposed application.
 **Inference with Hugging Face library**
 
 ```python
-import transformers
 import torch
+import transformers
 
-model_path = 'microsoft/Orca-2-7b'
-device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
-model = transformers.AutoModelForCausalLM.from_pretrained(model_path)
-model.to(device)
+if torch.cuda.is_available():
+    torch.set_default_device("cuda")
+else:
+    torch.set_default_device("cpu")
+    
+model = transformers.AutoModelForCausalLM.from_pretrained("microsoft/Orca-2-7b", device_map='auto')
 
+# https://github.com/huggingface/transformers/issues/27132
+# please use the slow tokenizer since fast and slow tokenizer produces different tokens
 tokenizer = transformers.AutoTokenizer.from_pretrained(
-    model_path,
-    model_max_length=4096,
-    padding_side="right",
-    use_fast=False,
-    add_special_tokens=False,
-)
+        "microsoft/Orca-2-7b",
+        use_fast=False,
+    )
 
 system_message = "You are Orca, an AI language model created by Microsoft. You are a cautious assistant. You carefully follow instructions. You are helpful and harmless and you follow ethical guidelines and promote positive behavior."
-user_message = "\" \n :You can't just say, \"\"that's crap\"\" and remove it without gaining a consensus. You already know this, based on your block history. —/ \" \nIs the comment obscene? \nOptions : Yes, No."
+user_message = "How can you determine if a restaurant is popular among locals or mainly attracts tourists, and why might this information be useful?"
 
-# We use Chat Markup Language https://github.com/MicrosoftDocs/azure-docs/blob/main/articles/ai-services/openai/includes/chat-markup-language.md#working-with-chat-markup-language-chatml    
-prompt =  f"<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{user_message}<|im_end|>\n<|im_start|>assistant"
+# We use Chat Markup Language https://github.com/MicrosoftDocs/azure-docs/blob/main/articles/ai-services/openai/includes/chat-markup-language.md#working-with-chat-markup-language-chatml
+prompt = f"<|im_start|>system\n{system_message}<|im_end|>\n<|im_start|>user\n{user_message}<|im_end|>\n<|im_start|>assistant"
 
 inputs = tokenizer(prompt, return_tensors='pt')
-inputs = inputs.to(device)
+output_ids = model.generate(inputs["input_ids"],)
+answer = tokenizer.batch_decode(output_ids)[0]
 
-output_ids = model.generate(inputs["input_ids"], max_length=4096, do_sample=False, temperature=0.0, use_cache=True)
-sequence_length = inputs["input_ids"].shape[1]
-new_output_ids = output_ids[:, sequence_length:]
-answers = tokenizer.batch_decode(new_output_ids, skip_special_tokens=True)
+print(answer)
+
+# This example continues showing how to add a second turn message by the user to the conversation
+second_turn_user_message = "Give me a list of the key points of your first answer."
+
+# we set add_special_tokens=False because we dont want to automatically add a bos_token between messages
+second_turn_message_in_markup = f"\n<|im_start|>user\n{second_turn_user_message}<|im_end|>\n<|im_start|>assistant"
+second_turn_tokens = tokenizer(second_turn_message_in_markup, return_tensors='pt', add_special_tokens=False)
+second_turn_input = torch.cat([output_ids, second_turn_tokens['input_ids']], dim=1)
+
+output_ids_2 = model.generate(second_turn_input,)
+second_turn_answer = tokenizer.batch_decode(output_ids_2)[0]
 
-print(answers[0])
+print(second_turn_answer)
 ```