File size: 1,976 Bytes
e3dce0b f1b2fc3 8d7cf56 e3dce0b 16242e1 e3dce0b b0f0f9e e3dce0b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
import os
import gradio as gr
import torch
import spaces
from transformers import AutoModelForCausalLM, AutoTokenizer
# Load model and tokenizer if a GPU is available
if torch.cuda.is_available():
model_id = "allenai/OLMo-7B-Instruct"
model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
else:
raise EnvironmentError("CUDA device not available. Please run on a GPU-enabled environment.")
# Basic function to generate response based on passage and question
@spaces.GPU
def generate_response(passage: str, question: str) -> str:
# Prepare the input text by combining the passage and question
messages = [{"role": "user", "content": f"Passage: {passage}\nQuestion: {question}"}]
inputs = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt")
# Generate text, focusing only on the new tokens added by the model
outputs = model.generate(**inputs, max_new_tokens=150)
# Decode only the generated part, skipping the prompt input
# generated_tokens = outputs[0][inputs.input_ids.shape[-1]:] # Ignore input tokens in the output
response = tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]
return response
# Gradio Interface
with gr.Blocks() as demo:
gr.Markdown("# Passage and Question Response Generator")
passage_input = gr.Textbox(label="Passage", placeholder="Enter the passage here", lines=5)
question_input = gr.Textbox(label="Question", placeholder="Enter the question here", lines=2)
output_box = gr.Textbox(label="Response", placeholder="Model's response will appear here")
submit_button = gr.Button("Generate Response")
submit_button.click(fn=generate_response, inputs=[passage_input, question_input], outputs=output_box)
# Run the app
if __name__ == "__main__":
demo.launch()
|