import os import torch import sys import gradio as gr import random from configs.infer_config import get_parser from huggingface_hub import hf_hub_download sys.path.append('./extern/dust3r') from dust3r.inference import inference, load_model from omegaconf import OmegaConf from pytorch_lightning import seed_everything from utils.diffusion_utils import instantiate_from_config,load_model_checkpoint,image_guided_synthesis import torchvision.transforms as transforms import copy i2v_examples = [ ['test/images/boy.png', 0, 1.0, '0 40', '0 0', '0 0', 50, 123], ['test/images/car.jpeg', 0, 1.0, '0 -35', '0 0', '0 -0.1', 50, 123], ['test/images/fruit.jpg', 0, 1.0, '0 -3 -15 -20 -17 -5 0', '0 -2 -5 -10 -8 -5 0 2 5 3 0', '0 0', 50, 123], ['test/images/room.png', 5, 1.0, '0 3 10 20 17 10 0', '0 -2 -8 -6 0 2 5 3 0', '0 -0.02 -0.09 -0.16 -0.09 0', 50, 123], ['test/images/castle.png', 0, 1.0, '0 30', '0 -1 -5 -4 0 1 5 4 0', '0 -0.2', 50, 123], ] max_seed = 2 ** 31 def download_model(): REPO_ID = 'Drexubery/ViewCrafter_25' filename_list = ['model.ckpt'] for filename in filename_list: local_file = os.path.join('./checkpoints/', filename) if not os.path.exists(local_file): hf_hub_download(repo_id=REPO_ID, filename=filename, local_dir='./checkpoints/', force_download=True) download_model() css = """#input_img {max-width: 1024px !important} #output_vid {max-width: 1024px; max-height:576px} #random_button {max-width: 100px !important}""" parser = get_parser() # infer_config.py opts = parser.parse_args() # default device: 'cuda:0' opts.save_dir = './' os.makedirs(opts.save_dir,exist_ok=True) test_tensor = torch.Tensor([0]).cuda() opts.device = str(test_tensor.device) dust3r = load_model(opts.model_path, opts.device) config = OmegaConf.load(opts.config) model_config = config.pop("model", OmegaConf.create()) model_config['params']['unet_config']['params']['use_checkpoint'] = False model = instantiate_from_config(model_config) model = model.to(opts.device) model.cond_stage_model.device = opts.device model.perframe_ae = opts.perframe_ae assert os.path.exists(opts.ckpt_path), "Error: checkpoint Not Found!" model = load_model_checkpoint(model, opts.ckpt_path) model.eval() diffusion = model transform = transforms.Compose([ transforms.Resize(576), transforms.CenterCrop((576,1024)), ]) def infer(opts,i2v_input_image, i2v_elevation, i2v_center_scale, i2v_d_phi, i2v_d_theta, i2v_d_r, i2v_steps, i2v_seed): elevation = float(i2v_elevation) center_scale = float(i2v_center_scale) ddim_steps = i2v_steps gradio_traj = [float(i) for i in i2v_d_phi.split()],[float(i) for i in i2v_d_theta.split()],[float(i) for i in i2v_d_r.split()] seed_everything(i2v_seed) torch.cuda.empty_cache() img_tensor = torch.from_numpy(i2v_input_image).permute(2, 0, 1).unsqueeze(0).float().to(self.device) img_tensor = (img_tensor / 255. - 0.5) * 2 image_tensor_resized = transform(img_tensor) #1,3,h,w images = get_input_dict(image_tensor_resized,idx = 0,dtype = torch.float32) images = [images, copy.deepcopy(images)] images[1]['idx'] = 1 se_images = images se_img_ori = (image_tensor_resized.squeeze(0).permute(1,2,0) + 1.)/2. run_dust3r(input_images=self.images) nvs_single_view(gradio=True) traj_dir = os.path.join(self.opts.save_dir, "viz_traj.mp4") gen_dir = os.path.join(self.opts.save_dir, "diffusion0.mp4") return i2v_traj_path,i2v_output_path with gr.Blocks(analytics_enabled=False, css=css) as viewcrafter_iface: gr.Markdown("