taesiri commited on
Commit
7183fd3
1 Parent(s): aaad2fd

initial commit

Browse files
Files changed (4) hide show
  1. .gitattributes +1 -0
  2. app.py +59 -0
  3. merlion.png +3 -0
  4. requirements.txt +3 -0
.gitattributes CHANGED
@@ -32,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
32
  *.zip filter=lfs diff=lfs merge=lfs -text
33
  *.zst filter=lfs diff=lfs merge=lfs -text
34
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
32
  *.zip filter=lfs diff=lfs merge=lfs -text
33
  *.zst filter=lfs diff=lfs merge=lfs -text
34
  *tfevents* filter=lfs diff=lfs merge=lfs -text
35
+ merlion.png filter=lfs diff=lfs merge=lfs -text
app.py ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import torch
3
+ from PIL import Image
4
+ import numpy as np
5
+ from PIL import Image
6
+ from lavis.models import load_model_and_preprocess
7
+ import gradio as gr
8
+
9
+ device = torch.device("cuda") if torch.cuda.is_available() else "cpu"
10
+
11
+
12
+ model, vis_processors, _ = load_model_and_preprocess(
13
+ name="blip2_opt", model_type="pretrain_opt2.7b", is_eval=True, device=device
14
+ )
15
+
16
+
17
+ def answer_question(image, prompt):
18
+ image = vis_processors["eval"](image).unsqueeze(0).to(device)
19
+ response = model.generate({"image": image, "prompt": f"Question: {prompt} Answer:"})
20
+ response = '\n'.join(response)
21
+ return response
22
+
23
+ def generate_caption(image, caption_type):
24
+ image = vis_processors["eval"](image).unsqueeze(0).to(device)
25
+
26
+ if caption_type == "Beam Search":
27
+ caption = model.generate({"image": image})
28
+ else:
29
+ caption = model.generate({"image": image}, use_nucleus_sampling=True, num_captions=3)
30
+
31
+ caption = '\n'.join(caption)
32
+
33
+ return caption
34
+
35
+
36
+ with gr.Blocks() as demo:
37
+
38
+ gr.Markdown("## BLIP-2 Demo")
39
+ gr.Markdown("Using `OPT2.7B` - [Github](https://github.com/salesforce/LAVIS/tree/main/projects/blip2) - [Paper](https://arxiv.org/abs/2301.12597)")
40
+
41
+
42
+ with gr.Row():
43
+ with gr.Column():
44
+ input_image = gr.Image(label="Image", type="pil")
45
+ caption_type = gr.Radio(["Beam Search", "Nucleus Sampling"], label="Caption Type", value="Beam Search")
46
+ btn_caption = gr.Button("Generate Caption")
47
+
48
+ question_txt = gr.Textbox(label="Question", lines=1)
49
+ btn_answer = gr.Button("Generate Answer")
50
+
51
+ with gr.Column():
52
+ output_text = gr.Textbox(label="Answer", lines=5)
53
+
54
+ btn_caption.click(generate_caption, inputs=[input_image, caption_type], outputs=[output_text])
55
+ btn_answer.click(answer_question, inputs=[input_image, question_txt], outputs=[output_text])
56
+
57
+ gr.Examples([['./merlion.png', 'Beam Search', 'which city is this?']], inputs=[input_image, caption_type, question_txt])
58
+
59
+ demo.launch()
merlion.png ADDED

Git LFS Details

  • SHA256: f1f3b6a507ec92e8f47ac6d7c64e11b03fcba8c550bcb6851f80e261e8951431
  • Pointer size: 132 Bytes
  • Size of remote file: 1.6 MB
requirements.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ torch
2
+ torchvision
3
+ salesforce-lavis