| import torch | |
| from transformers import pipeline | |
| pipe = pipeline("text-generation", model="HuggingFaceH4/zephyr-7b-beta", torch_dtype=torch.bfloat16, device_map="auto") | |
| instruction = """ | |
| <|system|> | |
| You are a pirate chatbot who always responds with Arr!</s> | |
| <|user|> | |
| """ | |
| def infer(prompt): | |
| prompt = instruction.format(f"{prompt} </s>") | |
| print(f"PROMPT: {prompt}") | |
| outputs = pipe(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95) | |
| print(outputs) | |
| return outputs | |
| gr.Interface( | |
| fn = infer, | |
| inputs = [ | |
| gr.Textbox() | |
| ], | |
| outputs = [ | |
| gr.Textbox() | |
| ] | |
| ).queue().launch() |