Upload 2 files

app.py

@@ -0,0 +1,46 @@
+import os
+import gradio as gr
+import torch
+import spaces
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+# Load model and tokenizer if a GPU is available
+if torch.cuda.is_available():
+    model_id = "allenai/OLMo-7B-Instruct"
+    model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", trust_remote_code=True)
+    tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
+else:
+    raise EnvironmentError("CUDA device not available. Please run on a GPU-enabled environment.")
+
+# Basic function to generate response based on passage and question
+@spaces.GPU
+def generate_response(passage: str, question: str) -> str:
+    # Prepare the input text by combining the passage and question
+    user_message = f"Passage: {passage}\nQuestion: {question}"
+    inputs = tokenizer(user_message, return_tensors="pt").to(model.device)
+    
+    # Generate text, focusing only on the new tokens added by the model
+    outputs = model.generate(inputs.input_ids, max_new_tokens=150)
+    
+    # Decode only the generated part, skipping the prompt input
+    generated_tokens = outputs[0][inputs.input_ids.shape[-1]:]  # Ignore input tokens in the output
+    response = tokenizer.decode(generated_tokens, skip_special_tokens=True)
+    
+    return response
+
+
+# Gradio Interface
+with gr.Blocks() as demo:
+    gr.Markdown("# Passage and Question Response Generator")
+    
+    passage_input = gr.Textbox(label="Passage", placeholder="Enter the passage here", lines=5)
+    question_input = gr.Textbox(label="Question", placeholder="Enter the question here", lines=2)
+    
+    output_box = gr.Textbox(label="Response", placeholder="Model's response will appear here")
+    
+    submit_button = gr.Button("Generate Response")
+    submit_button.click(fn=generate_response, inputs=[passage_input, question_input], outputs=output_box)
+
+# Run the app
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -0,0 +1,2 @@
+ai2-olmo
+accelerate