Update app.py

app.py

@@ -28,8 +28,7 @@ processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True)
 
 model = AutoModelForCausalLM.from_pretrained(
     model_id,
-    torch_dtype=torch.float16,
-    low_cpu_mem_usage=True,
+    torch_dtype="auto",
     trust_remote_code=True,
 )
 
@@ -59,8 +58,18 @@ def bot_streaming(message, history):
         # Handle the case where 'image' is not defined at all
         gr.Error("You need to upload an image for Phi-3-vision to work.")
 
-    prompt = f"{message['text']}<|image_1|>\nCan you convert the table to markdown format?{prompt_suffix}{assistant_prompt}"
-    print(f"prompt: {prompt}")
+    # prompt = f"{message['text']}<|image_1|>\nCan you convert the table to markdown format?{prompt_suffix}{assistant_prompt}"
+    chat = [
+        {"role": "user", "content": f"<|image_1|>\n{message['text']}"},
+    ]
+    prompt = processor.tokenizer.apply_chat_template(chat, tokenize=False, add_generation_prompt=True)
+
+    # need to remove last <|endoftext|> if it is there, which is used for training, not inference. For training, make sure to add <|endoftext|> in the end.
+    if prompt.endswith("<|endoftext|>"):
+        prompt = prompt.rstrip("<|endoftext|>")
+    
+    print(f">>> Prompt\n{prompt}")}")
+    
     image = Image.open(image)
     inputs = processor(prompt, [image], return_tensors='pt').to("cuda:0")