app.py

import gradio as gr  
import torch  
from transformers import AutoTokenizer  
from open_lm.utils.transformers.hf_config import OpenLMConfig  
from open_lm.utils.transformers.hf_model import OpenLMforCausalLM

title = """# 🙋🏻‍♂️ Welcome to Tonic's DCLM 1B"""


# Load the model and tokenizer  
model_name = "TRI-ML/DCLM-1B-IT"  
  
# Load the configuration, tokenizer, and model separately  
config = OpenLMConfig.from_pretrained(model_name)  
tokenizer = AutoTokenizer.from_pretrained(model_name)  
model = OpenLMforCausalLM.from_pretrained(model_name, torch_dtype=torch.bfloat16, device_map="cuda", config=config  )
  
# Define the prompt format  
def create_prompt(instruction):  
    PROMPT = '''Below is an instruction that describes a task.\n\nWrite a response that appropriately completes the request.\n\n### Instruction:\n{instruction}\n\n### Response:'''  
    return PROMPT.format(instruction=instruction)  
  
# Define the respond function for Gradio  
def respond(message, history, system_message, max_tokens, temperature, top_p):  
    # Create the prompt  
    prompt = create_prompt(message)  
      
    # Tokenize the input  
    input_ids = tokenizer.encode(prompt, return_tensors="pt").to(torch.device('cuda'))  
      
    # Generate the response  
    output = model.generate(input_ids, max_length=max_tokens, top_p=top_p, do_sample=True, temperature=temperature)  
      
    # Decode the response  
    response = tokenizer.decode(output[0][len(input_ids[0]):])  
    response = response.split("<|endoftext|>")[0]  
      
    return response  
  
# Create Gradio ChatInterface  
demo = gr.ChatInterface(
    gr.markdown(title),
    # gr.markdown(description),
    respond,  
    additional_inputs=[  
        # gr.Textbox(value="You are a friendly Chatbot.", label="System message"),  
        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),  
        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),  
        gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)")  
    ],  
)  
  
if __name__ == "__main__":  
    demo.launch()