Update app.py

app.py

@@ -1,22 +1,11 @@
-import requests
-from PIL import Image
-from transformers import BlipProcessor, BlipForConditionalGeneration
+import torch
+from diffusers import DiffusionPipeline, DPMSolverMultistepScheduler
+from diffusers.utils import export_to_video
 
-processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-large")
-model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large")
+pipe = DiffusionPipeline.from_pretrained("damo-vilab/text-to-video-ms-1.7b", torch_dtype=torch.float16, variant="fp16")
+pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
+pipe.enable_model_cpu_offload()
 
-img_url = 'https://storage.googleapis.com/sfr-vision-language-research/BLIP/demo.jpg' 
-raw_image = Image.open(requests.get(img_url, stream=True).raw).convert('RGB')
-
-# conditional image captioning
-text = "a photography of"
-inputs = processor(raw_image, text, return_tensors="pt")
-
-out = model.generate(**inputs)
-print(processor.decode(out[0], skip_special_tokens=True))
-
-# unconditional image captioning
-inputs = processor(raw_image, return_tensors="pt")
-
-out = model.generate(**inputs)
-print(processor.decode(out[0], skip_special_tokens=True))
+prompt = "Spiderman is surfing"
+video_frames = pipe(prompt, num_inference_steps=25).frames
+video_path = export_to_video(video_frames)