Update app.py

app.py

@@ -14,9 +14,10 @@ import gradio as gr
 from threading import Thread
 
 HF_TOKEN = os.environ.get("HF_TOKEN", None)
-MODEL = "NousResearch/Hermes-3-Llama-3.2-3B"
+MODEL = "evabyte/EvaByte-SFT"
+MODEL_BASE = "evabyte/EvaByte"
 
-TITLE = "<h1><center>Hermes-3-Llama-3.2-3B</center></h1>"
+TITLE = "<h1><center>EvaByte</center></h1>"
 
 PLACEHOLDER = """
 <center>
@@ -45,13 +46,14 @@ quantization_config = BitsAndBytesConfig(
     bnb_4bit_use_double_quant=True,
     bnb_4bit_quant_type= "nf4")
 
-tokenizer = AutoTokenizer.from_pretrained(MODEL)
+tokenizer = AutoTokenizer.from_pretrained(MODEL, trust_remote_code=True)
 model = AutoModelForCausalLM.from_pretrained(
     MODEL,
     torch_dtype=torch.float16,
     device_map="auto",
+    trust_remote_code=True,
     attn_implementation="flash_attention_2",
-    quantization_config=quantization_config)
+    quantization_config=quantization_config).eval().to(device)
 
 # Ensure `pad_token_id` is set
 if tokenizer.pad_token_id is None:
@@ -82,7 +84,7 @@ def stream_chat(
 
     conversation.append({"role": "user", "content": message})
 
-    input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt").to(model.device)
+    input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt").to(device)
     
     streamer = TextIteratorStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True)