Update app.py

app.py

@@ -30,27 +30,31 @@ model = AutoModelForVision2Seq.from_pretrained(model_id, torch_dtype="auto", dev
 
 @spaces.GPU
 def stream_chat(message: Dict[str, str], history: list):
-    print(message)
+    # {'text': 'what is this', 'files': ['image-xxx.jpg']}
+
+
+    image = Image.open(message["files"][0])
+    pixel_values = processor(images=[image], return_tensors="pt").to(model.device)["pixel_values"]
+
     conversation = []
     for prompt, answer in history:
         conversation.extend([{"role": "user", "content": prompt}, {"role": "assistant", "content": answer}])
 
-    conversation.append({"role": "user", "content": message})
+    conversation.append({"role": "user", "content": message["text"]})
+
+    input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt")
+    image_token_id = tokenizer.convert_tokens_to_ids("<image>")
+    image_prefix = torch.empty((1, getattr(processor, "image_seq_length")), dtype=input_ids.dtype).fill_(image_token_id)
+    input_ids = torch.cat((image_prefix, input_ids), dim=-1).to(model.device)
 
-    input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt").to(
-        model.device
-    )
     streamer = TextIteratorStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True)
 
     generate_kwargs = dict(
         input_ids=input_ids,
+        pixel_values=pixel_values
         streamer=streamer,
-        max_new_tokens=max_new_tokens,
-        temperature=temperature,
-        do_sample=True,
+        max_new_tokens=256,
     )
-    if temperature == 0:
-        generate_kwargs["do_sample"] = False
 
     t = Thread(target=model.generate, kwargs=generate_kwargs)
     t.start()