diff --git a/__pycache__/viewcrafter.cpython-39.pyc b/__pycache__/viewcrafter.cpython-39.pyc
index 4ac3f83d5cce7e8cda8ff31c60beae0b594048d9..4804c565dd05362f582234b90a4e34550ed2f62e 100644
Binary files a/__pycache__/viewcrafter.cpython-39.pyc and b/__pycache__/viewcrafter.cpython-39.pyc differ
diff --git a/app.py b/app.py
index b290a55844548639e5a733f229a178026b6ed7d3..cf6d026931ef17e68d1a7c11a2f3b55feec2bb21 100644
--- a/app.py
+++ b/app.py
@@ -1,7 +1,7 @@
import os
import torch
import sys
-import spaces
+import spaces #fixme
import gradio as gr
@@ -9,12 +9,20 @@ import random
from configs.infer_config import get_parser
from huggingface_hub import hf_hub_download
-i2v_examples = [
- ['test/images/boy.png', 0, 1.0, '0 40', '0 0', '0 0', 50, 123],
- ['test/images/car.jpeg', 0, 1.0, '0 -35', '0 0', '0 -0.1', 50, 123],
- ['test/images/fruit.jpg', 0, 1.0, '0 -3 -15 -20 -17 -5 0', '0 -2 -5 -10 -8 -5 0 2 5 3 0', '0 0', 50, 123],
- ['test/images/room.png', 5, 1.0, '0 3 10 20 17 10 0', '0 -2 -8 -6 0 2 5 3 0', '0 -0.02 -0.09 -0.16 -0.09 0', 50, 123],
- ['test/images/castle.png', 0, 1.0, '0 30', '0 -1 -5 -4 0 1 5 4 0', '0 -0.2', 50, 123],
+traj_examples = [
+ ['0 40', '0 0', '0 0'],
+ ['0 -35', '0 0', '0 -0.1'],
+ ['0 -3 -15 -20 -17 -5 0', '0 -2 -5 -10 -8 -5 0 2 5 3 0', '0 0'],
+ ['0 3 10 20 17 10 0', '0 -2 -8 -6 0 2 5 3 0', '0 -0.02 -0.09 -0.16 -0.09 0'],
+ ['0 30', '0 -1 -5 -4 0 1 5 4 0', '0 -0.2'],
+]
+
+img_examples = [
+ ['test/images/boy.png',0,1],
+ ['test/images/car.jpeg',5,1],
+ ['test/images/fruit.jpg',5,1],
+ ['test/images/room.png',10,1],
+ ['test/images/castle.png',-4,1],
]
max_seed = 2 ** 31
@@ -27,7 +35,7 @@ def download_model():
if not os.path.exists(local_file):
hf_hub_download(repo_id=REPO_ID, filename=filename, local_dir='./checkpoints/', force_download=True)
-download_model()
+download_model() #fixme
parser = get_parser() # infer_config.py
opts = parser.parse_args() # default device: 'cuda:0'
opts.save_dir = './'
@@ -35,7 +43,7 @@ os.makedirs(opts.save_dir,exist_ok=True)
test_tensor = torch.Tensor([0]).cuda()
opts.device = str(test_tensor.device)
-# install pytorch3d
+# install pytorch3d # fixme
pyt_version_str=torch.__version__.split("+")[0].replace(".", "")
version_str="".join([
f"py3{sys.version_info.minor}_cu",
@@ -50,10 +58,12 @@ print(f'>>> System info: {version_str}')
from viewcrafter import ViewCrafter
+
def viewcrafter_demo(opts):
css = """#input_img {max-width: 1024px !important} #output_vid {max-width: 1024px; max-height:576px} #random_button {max-width: 100px !important}"""
image2video = ViewCrafter(opts, gradio = True)
- image2video.run_gradio = spaces.GPU(image2video.run_gradio, duration=300)
+ image2video.run_traj = spaces.GPU(image2video.run_traj, duration=50) # fixme
+ image2video.run_gen = spaces.GPU(image2video.run_gen, duration=250) # fixme
with gr.Blocks(analytics_enabled=False, css=css) as viewcrafter_iface:
gr.Markdown("")
-
- #######image2video######
- with gr.Tab(label="ViewCrafter_25, 'single_view_txt' mode"):
- with gr.Column():
- with gr.Row():
- with gr.Column():
- with gr.Row():
- i2v_input_image = gr.Image(label="Input Image",elem_id="input_img")
- with gr.Row():
- i2v_elevation = gr.Slider(minimum=-45, maximum=45, step=1, elem_id="elevation", label="elevation", value=5)
- with gr.Row():
- i2v_center_scale = gr.Slider(minimum=0.1, maximum=2, step=0.1, elem_id="i2v_center_scale", label="center_scale", value=1)
- with gr.Row():
- i2v_d_phi = gr.Text(label='d_phi sequence, should start with 0')
- with gr.Row():
- i2v_d_theta = gr.Text(label='d_theta sequence, should start with 0')
- with gr.Row():
- i2v_d_r = gr.Text(label='d_r sequence, should start with 0')
- with gr.Row():
- i2v_steps = gr.Slider(minimum=1, maximum=50, step=1, elem_id="i2v_steps", label="Sampling steps", value=50)
- with gr.Row():
- i2v_seed = gr.Slider(label='Random Seed', minimum=0, maximum=max_seed, step=1, value=123)
- i2v_end_btn = gr.Button("Generate")
- # with gr.Tab(label='Result'):
- with gr.Column():
- with gr.Row():
- i2v_traj_video = gr.Video(label="Camera Trajectory",elem_id="traj_vid",autoplay=True,show_share_button=True)
- with gr.Row():
- i2v_output_video = gr.Video(label="Generated Video",elem_id="output_vid",autoplay=True,show_share_button=True)
-
- gr.Examples(examples=i2v_examples,
- inputs=[i2v_input_image, i2v_elevation, i2v_center_scale, i2v_d_phi, i2v_d_theta, i2v_d_r, i2v_steps, i2v_seed],
- outputs=[i2v_traj_video,i2v_output_video],
- fn = image2video.run_gradio,
- cache_examples='lazy',
- )
-
- # image2video.run_gradio(i2v_input_image='test/images/boy.png', i2v_elevation='10', i2v_d_phi='0 40', i2v_d_theta='0 0', i2v_d_r='0 0', i2v_center_scale=1, i2v_steps=50, i2v_seed=123)
- i2v_end_btn.click(inputs=[i2v_input_image, i2v_elevation, i2v_center_scale, i2v_d_phi, i2v_d_theta, i2v_d_r, i2v_steps, i2v_seed],
- outputs=[i2v_traj_video,i2v_output_video],
- fn = image2video.run_gradio
+ [Github] \
+ [Video] ")
+
+
+ with gr.Column():
+ # step 0: tutorial
+ gr.Markdown("## Step 0: Read tutorial", show_label=False)
+ gr.Markdown("Please refer to the tutorial
here for best practice, which includes the cameara system defination and the renderer parameters.
1. Estimate an elevation angle that represents the angle at which the image was taken; a value bigger than 0 indicates a top-down view, and it doesn't need to be precise.
2. The origin of the world coordinate system is by default defined at the point cloud corresponding to the center pixel of the input image. You can adjust the position of the origin by modifying center_scale; a value smaller than 1 brings the origin closer to you.
")
+ with gr.Row(equal_height=True):
+ with gr.Column(scale=2):
+ with gr.Row():
+ i2v_input_image = gr.Image(label="Input Image",elem_id="input_img")
+ with gr.Row():
+ i2v_elevation = gr.Slider(minimum=-45, maximum=45, step=1, elem_id="elevation", label="elevation", value=5)
+ i2v_center_scale = gr.Slider(minimum=0.1, maximum=2, step=0.1, elem_id="i2v_center_scale", label="center_scale", value=1)
+ gr.Examples(examples=img_examples,
+ inputs=[i2v_input_image,i2v_elevation,i2v_center_scale],
+ examples_per_page=6
)
+ # step 2 - camera trajectory generation
+ gr.Markdown("---\n## Step 2: Input camera trajectory", show_label=False, visible=True)
+ gr.Markdown(" Input a d_phi sequence, a d_theta sequence, and a d_r sequence to generate a camera trajectory. In the sequences, a positive d_phi moves the camera to the right, a negative d_theta moves the camera up, and a negative d_r moves the camera forward. Ensure that each sequence starts with 0 and contains at least two elements (a start and an end).
")
+ with gr.Row():
+ with gr.Column():
+ # camera_mode = gr.Radio(choices=CAMERA_MOTION_MODE, value=CAMERA_MOTION_MODE[0], label="Camera Motion Control Mode", interactive=True, visible=False)
+ i2v_d_phi = gr.Text(label='d_phi sequence')
+ i2v_d_theta = gr.Text(label='d_theta sequence')
+ i2v_d_r = gr.Text(label='d_r sequence')
+ i2v_start_btn = gr.Button("Generate trajectory")
+ # camera_info = gr.Button(value="Proceed", visible=False)
+ with gr.Column():
+ i2v_traj_video = gr.Video(label="Camera Trajectory",elem_id="traj_vid",autoplay=True,show_share_button=True)
+
+ gr.Examples(examples=traj_examples,
+ inputs=[i2v_d_phi, i2v_d_theta, i2v_d_r],
+ )
+
+ # step 3 - Generate video
+ gr.Markdown("---\n## Step 3: Generate video", show_label=False, visible=True)
+ gr.Markdown(" You can reduce the sampling steps for faster inference; try different random seed if the result is not satisfying.
")
+ with gr.Row():
+ with gr.Column():
+ i2v_steps = gr.Slider(minimum=1, maximum=50, step=1, elem_id="i2v_steps", label="Sampling steps", value=50)
+ i2v_seed = gr.Slider(label='Random seed', minimum=0, maximum=max_seed, step=1, value=123)
+ i2v_end_btn = gr.Button("Generate video")
+ # with gr.Tab(label='Result'):
+ with gr.Column():
+ i2v_output_video = gr.Video(label="Generated Video",elem_id="output_vid",autoplay=True,show_share_button=True)
+
+
+
+ i2v_start_btn.click(inputs=[i2v_input_image, i2v_elevation, i2v_center_scale, i2v_d_phi, i2v_d_theta, i2v_d_r],
+ outputs=[i2v_traj_video],
+ fn = image2video.run_traj
+ )
+
+ i2v_end_btn.click(inputs=[i2v_steps, i2v_seed],
+ outputs=[i2v_output_video],
+ fn = image2video.run_gen
+ )
return viewcrafter_iface
viewcrafter_iface = viewcrafter_demo(opts)
viewcrafter_iface.queue(max_size=10)
-viewcrafter_iface.launch()
-# viewcrafter_iface.launch(server_name='127.0.0.1', server_port=80, max_threads=1,debug=False)
+viewcrafter_iface.launch() #fixme
+# viewcrafter_iface.launch(server_name='11.220.92.96', server_port=80, max_threads=1,debug=False)
diff --git a/app_old.py b/app_old.py
new file mode 100644
index 0000000000000000000000000000000000000000..b290a55844548639e5a733f229a178026b6ed7d3
--- /dev/null
+++ b/app_old.py
@@ -0,0 +1,122 @@
+import os
+import torch
+import sys
+import spaces
+
+
+import gradio as gr
+import random
+from configs.infer_config import get_parser
+from huggingface_hub import hf_hub_download
+
+i2v_examples = [
+ ['test/images/boy.png', 0, 1.0, '0 40', '0 0', '0 0', 50, 123],
+ ['test/images/car.jpeg', 0, 1.0, '0 -35', '0 0', '0 -0.1', 50, 123],
+ ['test/images/fruit.jpg', 0, 1.0, '0 -3 -15 -20 -17 -5 0', '0 -2 -5 -10 -8 -5 0 2 5 3 0', '0 0', 50, 123],
+ ['test/images/room.png', 5, 1.0, '0 3 10 20 17 10 0', '0 -2 -8 -6 0 2 5 3 0', '0 -0.02 -0.09 -0.16 -0.09 0', 50, 123],
+ ['test/images/castle.png', 0, 1.0, '0 30', '0 -1 -5 -4 0 1 5 4 0', '0 -0.2', 50, 123],
+]
+
+max_seed = 2 ** 31
+
+def download_model():
+ REPO_ID = 'Drexubery/ViewCrafter_25'
+ filename_list = ['model.ckpt']
+ for filename in filename_list:
+ local_file = os.path.join('./checkpoints/', filename)
+ if not os.path.exists(local_file):
+ hf_hub_download(repo_id=REPO_ID, filename=filename, local_dir='./checkpoints/', force_download=True)
+
+download_model()
+parser = get_parser() # infer_config.py
+opts = parser.parse_args() # default device: 'cuda:0'
+opts.save_dir = './'
+os.makedirs(opts.save_dir,exist_ok=True)
+test_tensor = torch.Tensor([0]).cuda()
+opts.device = str(test_tensor.device)
+
+# install pytorch3d
+pyt_version_str=torch.__version__.split("+")[0].replace(".", "")
+version_str="".join([
+ f"py3{sys.version_info.minor}_cu",
+ torch.version.cuda.replace(".",""),
+ f"_pyt{pyt_version_str}"
+])
+print(version_str)
+os.system(f"{sys.executable} -m pip install --no-index --no-cache-dir pytorch3d -f https://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/{version_str}/download.html")
+os.system("mkdir -p checkpoints/ && wget https://download.europe.naverlabs.com/ComputerVision/DUSt3R/DUSt3R_ViTLarge_BaseDecoder_512_dpt.pth -P checkpoints/")
+print(f'>>> System info: {version_str}')
+
+
+from viewcrafter import ViewCrafter
+
+def viewcrafter_demo(opts):
+ css = """#input_img {max-width: 1024px !important} #output_vid {max-width: 1024px; max-height:576px} #random_button {max-width: 100px !important}"""
+ image2video = ViewCrafter(opts, gradio = True)
+ image2video.run_gradio = spaces.GPU(image2video.run_gradio, duration=300)
+ with gr.Blocks(analytics_enabled=False, css=css) as viewcrafter_iface:
+ gr.Markdown("")
+
+ #######image2video######
+ with gr.Tab(label="ViewCrafter_25, 'single_view_txt' mode"):
+ with gr.Column():
+ with gr.Row():
+ with gr.Column():
+ with gr.Row():
+ i2v_input_image = gr.Image(label="Input Image",elem_id="input_img")
+ with gr.Row():
+ i2v_elevation = gr.Slider(minimum=-45, maximum=45, step=1, elem_id="elevation", label="elevation", value=5)
+ with gr.Row():
+ i2v_center_scale = gr.Slider(minimum=0.1, maximum=2, step=0.1, elem_id="i2v_center_scale", label="center_scale", value=1)
+ with gr.Row():
+ i2v_d_phi = gr.Text(label='d_phi sequence, should start with 0')
+ with gr.Row():
+ i2v_d_theta = gr.Text(label='d_theta sequence, should start with 0')
+ with gr.Row():
+ i2v_d_r = gr.Text(label='d_r sequence, should start with 0')
+ with gr.Row():
+ i2v_steps = gr.Slider(minimum=1, maximum=50, step=1, elem_id="i2v_steps", label="Sampling steps", value=50)
+ with gr.Row():
+ i2v_seed = gr.Slider(label='Random Seed', minimum=0, maximum=max_seed, step=1, value=123)
+ i2v_end_btn = gr.Button("Generate")
+ # with gr.Tab(label='Result'):
+ with gr.Column():
+ with gr.Row():
+ i2v_traj_video = gr.Video(label="Camera Trajectory",elem_id="traj_vid",autoplay=True,show_share_button=True)
+ with gr.Row():
+ i2v_output_video = gr.Video(label="Generated Video",elem_id="output_vid",autoplay=True,show_share_button=True)
+
+ gr.Examples(examples=i2v_examples,
+ inputs=[i2v_input_image, i2v_elevation, i2v_center_scale, i2v_d_phi, i2v_d_theta, i2v_d_r, i2v_steps, i2v_seed],
+ outputs=[i2v_traj_video,i2v_output_video],
+ fn = image2video.run_gradio,
+ cache_examples='lazy',
+ )
+
+ # image2video.run_gradio(i2v_input_image='test/images/boy.png', i2v_elevation='10', i2v_d_phi='0 40', i2v_d_theta='0 0', i2v_d_r='0 0', i2v_center_scale=1, i2v_steps=50, i2v_seed=123)
+ i2v_end_btn.click(inputs=[i2v_input_image, i2v_elevation, i2v_center_scale, i2v_d_phi, i2v_d_theta, i2v_d_r, i2v_steps, i2v_seed],
+ outputs=[i2v_traj_video,i2v_output_video],
+ fn = image2video.run_gradio
+ )
+
+ return viewcrafter_iface
+
+
+viewcrafter_iface = viewcrafter_demo(opts)
+viewcrafter_iface.queue(max_size=10)
+viewcrafter_iface.launch()
+# viewcrafter_iface.launch(server_name='127.0.0.1', server_port=80, max_threads=1,debug=False)
+
diff --git a/configs/__pycache__/infer_config.cpython-39.pyc b/configs/__pycache__/infer_config.cpython-39.pyc
index 4b26bf11e42c7ac7df4a48e4e26373b2f4c9b167..3f4a1c55c59796bed82d416b2fd4f56594c24a94 100644
Binary files a/configs/__pycache__/infer_config.cpython-39.pyc and b/configs/__pycache__/infer_config.cpython-39.pyc differ
diff --git a/configs/inference_pvd_1024_local.yaml b/configs/inference_pvd_1024_local.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..42eac5217c75e099db62c21fc8c9da6e2089ecb9
--- /dev/null
+++ b/configs/inference_pvd_1024_local.yaml
@@ -0,0 +1,113 @@
+model:
+ pretrained_checkpoint: /apdcephfs_cq10/share_1290939/vg_share/vip3d_share/DC_1024/model.ckpt
+ base_learning_rate: 1.0e-05
+ scale_lr: False
+ target: lvdm.models.ddpm3d.VIPLatentDiffusion
+ params:
+ rescale_betas_zero_snr: True
+ parameterization: "v"
+ linear_start: 0.00085
+ linear_end: 0.012
+ num_timesteps_cond: 1
+ log_every_t: 200
+ timesteps: 1000
+ first_stage_key: video
+ cond_stage_key: caption
+ cond_stage_trainable: False
+ image_proj_model_trainable: True
+ conditioning_key: hybrid
+ image_size: [72, 128]
+ channels: 4
+ scale_by_std: False
+ scale_factor: 0.18215
+ use_ema: False
+ uncond_prob: 0.05
+ uncond_type: 'empty_seq'
+ rand_cond_frame: true
+ use_dynamic_rescale: true
+ base_scale: 0.3
+ fps_condition_type: 'fps'
+ perframe_ae: True
+ loop_video: Flase
+
+ unet_config:
+ target: lvdm.modules.networks.openaimodel3d.UNetModel
+ params:
+ in_channels: 8
+ out_channels: 4
+ model_channels: 320
+ attention_resolutions:
+ - 4
+ - 2
+ - 1
+ num_res_blocks: 2
+ channel_mult:
+ - 1
+ - 2
+ - 4
+ - 4
+ dropout: 0.1
+ num_head_channels: 64
+ transformer_depth: 1
+ context_dim: 1024
+ use_linear: true
+ use_checkpoint: True
+ temporal_conv: True
+ temporal_attention: True
+ temporal_selfatt_only: true
+ use_relative_position: false
+ use_causal_attention: False
+ temporal_length: 16
+ addition_attention: true
+ image_cross_attention: true
+ default_fs: 10
+ fs_condition: true
+
+ first_stage_config:
+ target: lvdm.models.autoencoder.AutoencoderKL
+ params:
+ embed_dim: 4
+ monitor: val/rec_loss
+ ddconfig:
+ double_z: True
+ z_channels: 4
+ resolution: 256
+ in_channels: 3
+ out_ch: 3
+ ch: 128
+ ch_mult:
+ - 1
+ - 2
+ - 4
+ - 4
+ num_res_blocks: 2
+ attn_resolutions: []
+ dropout: 0.0
+ lossconfig:
+ target: torch.nn.Identity
+
+ cond_stage_config:
+ target: lvdm.modules.encoders.condition.FrozenOpenCLIPEmbedder
+ params:
+ version: /apdcephfs_cq10/share_1290939/vg_share/vip3d_share/OpenCLIP-ViT-H-14-laion2B-s32B-b79K/blobs/9a78ef8e8c73fd0df621682e7a8e8eb36c6916cb3c16b291a082ecd52ab79cc4
+ freeze: true
+ layer: "penultimate"
+
+ img_cond_stage_config:
+ target: lvdm.modules.encoders.condition.FrozenOpenCLIPImageEmbedderV2
+ params:
+ version: /apdcephfs_cq10/share_1290939/vg_share/vip3d_share/OpenCLIP-ViT-H-14-laion2B-s32B-b79K/blobs/9a78ef8e8c73fd0df621682e7a8e8eb36c6916cb3c16b291a082ecd52ab79cc4
+ freeze: true
+
+ image_proj_stage_config:
+ target: lvdm.modules.encoders.resampler.Resampler
+ params:
+ dim: 1024
+ depth: 4
+ dim_head: 64
+ heads: 12
+ num_queries: 16
+ embedding_dim: 1280
+ output_dim: 1024
+ ff_mult: 4
+ video_length: 16
\ No newline at end of file
diff --git a/extern/dust3r/croco/models/__pycache__/blocks.cpython-39.pyc b/extern/dust3r/croco/models/__pycache__/blocks.cpython-39.pyc
index 94bc065a640fa83e39b03a0e5bd8c71e1e8781e7..9de043b772937eb8afaef293d4b16ed864f5349d 100644
Binary files a/extern/dust3r/croco/models/__pycache__/blocks.cpython-39.pyc and b/extern/dust3r/croco/models/__pycache__/blocks.cpython-39.pyc differ
diff --git a/extern/dust3r/croco/models/__pycache__/croco.cpython-39.pyc b/extern/dust3r/croco/models/__pycache__/croco.cpython-39.pyc
index 4fcdd77bdb8e949be52e82b04e40c85921427b20..6bf15fef72a85259476a278c6f38a529301ba5cd 100644
Binary files a/extern/dust3r/croco/models/__pycache__/croco.cpython-39.pyc and b/extern/dust3r/croco/models/__pycache__/croco.cpython-39.pyc differ
diff --git a/extern/dust3r/croco/models/__pycache__/dpt_block.cpython-39.pyc b/extern/dust3r/croco/models/__pycache__/dpt_block.cpython-39.pyc
index c1eb327ace6f4323a3222cdea1f373ba2de5c0e1..15f5d9aa7639dacde9802fb7eb037e7aa58eca26 100644
Binary files a/extern/dust3r/croco/models/__pycache__/dpt_block.cpython-39.pyc and b/extern/dust3r/croco/models/__pycache__/dpt_block.cpython-39.pyc differ
diff --git a/extern/dust3r/croco/models/__pycache__/masking.cpython-39.pyc b/extern/dust3r/croco/models/__pycache__/masking.cpython-39.pyc
index 7fe7317d48a0ce38df510617a65e0b65eb4ac608..ba92e047262a425fd5c9aabcb05034e17ff52fe1 100644
Binary files a/extern/dust3r/croco/models/__pycache__/masking.cpython-39.pyc and b/extern/dust3r/croco/models/__pycache__/masking.cpython-39.pyc differ
diff --git a/extern/dust3r/croco/models/__pycache__/pos_embed.cpython-39.pyc b/extern/dust3r/croco/models/__pycache__/pos_embed.cpython-39.pyc
index 2ef612e797edcbd6a284c18f11ce276bee2e8135..6910ff872aacee69971213f112563f9a8f255b53 100644
Binary files a/extern/dust3r/croco/models/__pycache__/pos_embed.cpython-39.pyc and b/extern/dust3r/croco/models/__pycache__/pos_embed.cpython-39.pyc differ
diff --git a/extern/dust3r/croco/models/curope/__pycache__/__init__.cpython-39.pyc b/extern/dust3r/croco/models/curope/__pycache__/__init__.cpython-39.pyc
index f261f30b323a7e69fe2419be79f313f871db6d87..a7c2e91bd0f26765b42b17fc4209fe659ba9e368 100644
Binary files a/extern/dust3r/croco/models/curope/__pycache__/__init__.cpython-39.pyc and b/extern/dust3r/croco/models/curope/__pycache__/__init__.cpython-39.pyc differ
diff --git a/extern/dust3r/croco/models/curope/__pycache__/curope2d.cpython-39.pyc b/extern/dust3r/croco/models/curope/__pycache__/curope2d.cpython-39.pyc
index 89b587d9c634f6b44623581fcfb357cae319e369..91fde824a039dded9d796b1f97fa677e5a0ca16e 100644
Binary files a/extern/dust3r/croco/models/curope/__pycache__/curope2d.cpython-39.pyc and b/extern/dust3r/croco/models/curope/__pycache__/curope2d.cpython-39.pyc differ
diff --git a/extern/dust3r/dust3r/__pycache__/__init__.cpython-39.pyc b/extern/dust3r/dust3r/__pycache__/__init__.cpython-39.pyc
index 5cbc4ce982cd261cc3c76b5608818ccb328a9f47..9af69e7a5c7c44b9c8fab61863b96f84ed96146b 100644
Binary files a/extern/dust3r/dust3r/__pycache__/__init__.cpython-39.pyc and b/extern/dust3r/dust3r/__pycache__/__init__.cpython-39.pyc differ
diff --git a/extern/dust3r/dust3r/__pycache__/image_pairs.cpython-39.pyc b/extern/dust3r/dust3r/__pycache__/image_pairs.cpython-39.pyc
index 2fa96923e752763f3ee527a2809edcef5514b9fc..45d7856a040a1aad61e49ba1fad124849b4c46df 100644
Binary files a/extern/dust3r/dust3r/__pycache__/image_pairs.cpython-39.pyc and b/extern/dust3r/dust3r/__pycache__/image_pairs.cpython-39.pyc differ
diff --git a/extern/dust3r/dust3r/__pycache__/inference.cpython-39.pyc b/extern/dust3r/dust3r/__pycache__/inference.cpython-39.pyc
index c14052435e7071ae255fb40bed026fc7996a40a9..6bdc63d641ac070a8183957427e67af9d3b5bc1e 100644
Binary files a/extern/dust3r/dust3r/__pycache__/inference.cpython-39.pyc and b/extern/dust3r/dust3r/__pycache__/inference.cpython-39.pyc differ
diff --git a/extern/dust3r/dust3r/__pycache__/model.cpython-39.pyc b/extern/dust3r/dust3r/__pycache__/model.cpython-39.pyc
index f8162a1c73264bf12dafb9a83b8c667e61f6194e..68dab80253726c8d491f790a2bbf36076fb58f90 100644
Binary files a/extern/dust3r/dust3r/__pycache__/model.cpython-39.pyc and b/extern/dust3r/dust3r/__pycache__/model.cpython-39.pyc differ
diff --git a/extern/dust3r/dust3r/__pycache__/optim_factory.cpython-39.pyc b/extern/dust3r/dust3r/__pycache__/optim_factory.cpython-39.pyc
index 7a35af6c2eef8251cab7ab5745ba5095739dac27..6d1d56b28709f9f7687eb143a310c60ebde2f399 100644
Binary files a/extern/dust3r/dust3r/__pycache__/optim_factory.cpython-39.pyc and b/extern/dust3r/dust3r/__pycache__/optim_factory.cpython-39.pyc differ
diff --git a/extern/dust3r/dust3r/__pycache__/patch_embed.cpython-39.pyc b/extern/dust3r/dust3r/__pycache__/patch_embed.cpython-39.pyc
index e21a13477cc1c69274d5163457409a2c5a7382cd..8e06cb617c7f5c49cd8c445c07acbc687605ca24 100644
Binary files a/extern/dust3r/dust3r/__pycache__/patch_embed.cpython-39.pyc and b/extern/dust3r/dust3r/__pycache__/patch_embed.cpython-39.pyc differ
diff --git a/extern/dust3r/dust3r/__pycache__/post_process.cpython-39.pyc b/extern/dust3r/dust3r/__pycache__/post_process.cpython-39.pyc
index 5b23e8f9bae33f6f8e1280bd1b759a8789309775..930502653f74310164737d35970bf751120813dc 100644
Binary files a/extern/dust3r/dust3r/__pycache__/post_process.cpython-39.pyc and b/extern/dust3r/dust3r/__pycache__/post_process.cpython-39.pyc differ
diff --git a/extern/dust3r/dust3r/__pycache__/viz.cpython-39.pyc b/extern/dust3r/dust3r/__pycache__/viz.cpython-39.pyc
index 8727cb04ab0fefbfedd696e5c8d1b9fd13f4fb78..8857e8a4f77c179ee129d23a80a9f1c54702e1c2 100644
Binary files a/extern/dust3r/dust3r/__pycache__/viz.cpython-39.pyc and b/extern/dust3r/dust3r/__pycache__/viz.cpython-39.pyc differ
diff --git a/extern/dust3r/dust3r/cloud_opt/__pycache__/__init__.cpython-39.pyc b/extern/dust3r/dust3r/cloud_opt/__pycache__/__init__.cpython-39.pyc
index 13a8b5d9d0a5d4d50898065efed58a0ef48512b9..b3f099bc02954b78acab5878b5c5b45a73e0be45 100644
Binary files a/extern/dust3r/dust3r/cloud_opt/__pycache__/__init__.cpython-39.pyc and b/extern/dust3r/dust3r/cloud_opt/__pycache__/__init__.cpython-39.pyc differ
diff --git a/extern/dust3r/dust3r/cloud_opt/__pycache__/base_opt.cpython-39.pyc b/extern/dust3r/dust3r/cloud_opt/__pycache__/base_opt.cpython-39.pyc
index e03428f374f4007f215bd47b75262b0efa44aa13..2cdaeca6cec5c7134420df968754a9602864ccc1 100644
Binary files a/extern/dust3r/dust3r/cloud_opt/__pycache__/base_opt.cpython-39.pyc and b/extern/dust3r/dust3r/cloud_opt/__pycache__/base_opt.cpython-39.pyc differ
diff --git a/extern/dust3r/dust3r/cloud_opt/__pycache__/commons.cpython-39.pyc b/extern/dust3r/dust3r/cloud_opt/__pycache__/commons.cpython-39.pyc
index 04765a70b3c62ac8ef8b00d99c0c4b4d0f38fcc7..8c4641157b5858dd6a887dda74911244c40ede69 100644
Binary files a/extern/dust3r/dust3r/cloud_opt/__pycache__/commons.cpython-39.pyc and b/extern/dust3r/dust3r/cloud_opt/__pycache__/commons.cpython-39.pyc differ
diff --git a/extern/dust3r/dust3r/cloud_opt/__pycache__/init_im_poses.cpython-39.pyc b/extern/dust3r/dust3r/cloud_opt/__pycache__/init_im_poses.cpython-39.pyc
index 1767601bf6949ea79e243027bf3784181ca31279..6f1275ad98b0f6dd9fe9b7bd74495d59020b617c 100644
Binary files a/extern/dust3r/dust3r/cloud_opt/__pycache__/init_im_poses.cpython-39.pyc and b/extern/dust3r/dust3r/cloud_opt/__pycache__/init_im_poses.cpython-39.pyc differ
diff --git a/extern/dust3r/dust3r/cloud_opt/__pycache__/optimizer.cpython-39.pyc b/extern/dust3r/dust3r/cloud_opt/__pycache__/optimizer.cpython-39.pyc
index ed8c9197e50172a3368545db8e927b052faa007f..f06dfe6f62fcfbcf4cd6f1d1f934ed6bd8a51980 100644
Binary files a/extern/dust3r/dust3r/cloud_opt/__pycache__/optimizer.cpython-39.pyc and b/extern/dust3r/dust3r/cloud_opt/__pycache__/optimizer.cpython-39.pyc differ
diff --git a/extern/dust3r/dust3r/cloud_opt/__pycache__/pair_viewer.cpython-39.pyc b/extern/dust3r/dust3r/cloud_opt/__pycache__/pair_viewer.cpython-39.pyc
index dac26bd3a0e9c80c912b6e0edadf8964b1fe8128..bf8b51320989f60f0c5050cadd4b5c407febc821 100644
Binary files a/extern/dust3r/dust3r/cloud_opt/__pycache__/pair_viewer.cpython-39.pyc and b/extern/dust3r/dust3r/cloud_opt/__pycache__/pair_viewer.cpython-39.pyc differ
diff --git a/extern/dust3r/dust3r/heads/__pycache__/__init__.cpython-39.pyc b/extern/dust3r/dust3r/heads/__pycache__/__init__.cpython-39.pyc
index 7fdada493920f46bb85186e612e104a2ec54e0ea..eb03236f5cc333749cd69931a204474f4b2a72b7 100644
Binary files a/extern/dust3r/dust3r/heads/__pycache__/__init__.cpython-39.pyc and b/extern/dust3r/dust3r/heads/__pycache__/__init__.cpython-39.pyc differ
diff --git a/extern/dust3r/dust3r/heads/__pycache__/dpt_head.cpython-39.pyc b/extern/dust3r/dust3r/heads/__pycache__/dpt_head.cpython-39.pyc
index d46437891d36993d117bc4af6bfdbf4694aca35f..4d01a4de060506ac87bef0f2fba8c81ff0dd7bc4 100644
Binary files a/extern/dust3r/dust3r/heads/__pycache__/dpt_head.cpython-39.pyc and b/extern/dust3r/dust3r/heads/__pycache__/dpt_head.cpython-39.pyc differ
diff --git a/extern/dust3r/dust3r/heads/__pycache__/linear_head.cpython-39.pyc b/extern/dust3r/dust3r/heads/__pycache__/linear_head.cpython-39.pyc
index 70b70a07356a477da201256f4248a35854d21d84..e116b1b14997425a75913df11d9ab17a65c55dbc 100644
Binary files a/extern/dust3r/dust3r/heads/__pycache__/linear_head.cpython-39.pyc and b/extern/dust3r/dust3r/heads/__pycache__/linear_head.cpython-39.pyc differ
diff --git a/extern/dust3r/dust3r/heads/__pycache__/postprocess.cpython-39.pyc b/extern/dust3r/dust3r/heads/__pycache__/postprocess.cpython-39.pyc
index 5bea21b74f2d9b97d60604f1a06fca917f19a53c..02231ee8ad397c2d96c39a795e37faf8cf4bd4cb 100644
Binary files a/extern/dust3r/dust3r/heads/__pycache__/postprocess.cpython-39.pyc and b/extern/dust3r/dust3r/heads/__pycache__/postprocess.cpython-39.pyc differ
diff --git a/extern/dust3r/dust3r/utils/__pycache__/__init__.cpython-39.pyc b/extern/dust3r/dust3r/utils/__pycache__/__init__.cpython-39.pyc
index d3b2c0fa22532edf8f70ad85c1f93988ad5561ad..69bfd865de76e853c220c0a11657be5849705593 100644
Binary files a/extern/dust3r/dust3r/utils/__pycache__/__init__.cpython-39.pyc and b/extern/dust3r/dust3r/utils/__pycache__/__init__.cpython-39.pyc differ
diff --git a/extern/dust3r/dust3r/utils/__pycache__/device.cpython-39.pyc b/extern/dust3r/dust3r/utils/__pycache__/device.cpython-39.pyc
index 0be0c0f0fa90cc061aecc93f40ff29966eb2e1cf..c653546402780d543bdc7d986113a0a1d92b809d 100644
Binary files a/extern/dust3r/dust3r/utils/__pycache__/device.cpython-39.pyc and b/extern/dust3r/dust3r/utils/__pycache__/device.cpython-39.pyc differ
diff --git a/extern/dust3r/dust3r/utils/__pycache__/geometry.cpython-39.pyc b/extern/dust3r/dust3r/utils/__pycache__/geometry.cpython-39.pyc
index ff6e9bdc95483f8346957d44c341a296b1214876..cc8042fd5ee830d52a6ff2b60c034d11ff5ac554 100644
Binary files a/extern/dust3r/dust3r/utils/__pycache__/geometry.cpython-39.pyc and b/extern/dust3r/dust3r/utils/__pycache__/geometry.cpython-39.pyc differ
diff --git a/extern/dust3r/dust3r/utils/__pycache__/image.cpython-39.pyc b/extern/dust3r/dust3r/utils/__pycache__/image.cpython-39.pyc
index 3f1db23940189e063812804f0589eb5686d174ba..c8a22f9ef8e047d493134449bd4df757f366e745 100644
Binary files a/extern/dust3r/dust3r/utils/__pycache__/image.cpython-39.pyc and b/extern/dust3r/dust3r/utils/__pycache__/image.cpython-39.pyc differ
diff --git a/extern/dust3r/dust3r/utils/__pycache__/misc.cpython-39.pyc b/extern/dust3r/dust3r/utils/__pycache__/misc.cpython-39.pyc
index 6f75578847e2c2c4d9e5363a4b8b2c07057df4b3..56057edd121c7c715bebc1f2e392cb2a9cf4d58a 100644
Binary files a/extern/dust3r/dust3r/utils/__pycache__/misc.cpython-39.pyc and b/extern/dust3r/dust3r/utils/__pycache__/misc.cpython-39.pyc differ
diff --git a/extern/dust3r/dust3r/utils/__pycache__/path_to_croco.cpython-39.pyc b/extern/dust3r/dust3r/utils/__pycache__/path_to_croco.cpython-39.pyc
index c3e9763a09e6057f8722f7428e9f84eba1d36443..b865c5fb14f2d9f5b9e90acf245ab0f8c2d3b38d 100644
Binary files a/extern/dust3r/dust3r/utils/__pycache__/path_to_croco.cpython-39.pyc and b/extern/dust3r/dust3r/utils/__pycache__/path_to_croco.cpython-39.pyc differ
diff --git a/lvdm/__pycache__/basics.cpython-39.pyc b/lvdm/__pycache__/basics.cpython-39.pyc
index 89ae5785bf4fa4b96a8ddacbfd421f0af29d2825..72d3ebd5e8d3c6d0fab6b09478e6703d5c49a282 100644
Binary files a/lvdm/__pycache__/basics.cpython-39.pyc and b/lvdm/__pycache__/basics.cpython-39.pyc differ
diff --git a/lvdm/__pycache__/common.cpython-39.pyc b/lvdm/__pycache__/common.cpython-39.pyc
index 48e9f286d22b8272372c3fa05970102a52d1fb36..d079f8fc7659c6fe9dc9f582f34a42b0b43c1cab 100644
Binary files a/lvdm/__pycache__/common.cpython-39.pyc and b/lvdm/__pycache__/common.cpython-39.pyc differ
diff --git a/lvdm/__pycache__/distributions.cpython-39.pyc b/lvdm/__pycache__/distributions.cpython-39.pyc
index a7da5ec8685a869b2b5b1726cd3e04a1e8c2eda4..9b304678455c724e0cf9a54618d8fc673e215f0d 100644
Binary files a/lvdm/__pycache__/distributions.cpython-39.pyc and b/lvdm/__pycache__/distributions.cpython-39.pyc differ
diff --git a/lvdm/__pycache__/ema.cpython-39.pyc b/lvdm/__pycache__/ema.cpython-39.pyc
index f8e7ee934b29f7f0299d9115b22fea894e21cd0b..f716ce11de2dcb222d9c36dca8e554a0954bb256 100644
Binary files a/lvdm/__pycache__/ema.cpython-39.pyc and b/lvdm/__pycache__/ema.cpython-39.pyc differ
diff --git a/lvdm/models/__pycache__/autoencoder.cpython-39.pyc b/lvdm/models/__pycache__/autoencoder.cpython-39.pyc
index 719168533d9969941cd6eb758b85422db522b407..36471118941bc9d9dcbd4d80020cacd2e5fbce47 100644
Binary files a/lvdm/models/__pycache__/autoencoder.cpython-39.pyc and b/lvdm/models/__pycache__/autoencoder.cpython-39.pyc differ
diff --git a/lvdm/models/__pycache__/ddpm3d.cpython-39.pyc b/lvdm/models/__pycache__/ddpm3d.cpython-39.pyc
index 1005a8e031ce15f3ddedc80b3edf01808e29703c..4f9d86fdc3b902eb4e56abb3dcb8d5d35163e7f4 100644
Binary files a/lvdm/models/__pycache__/ddpm3d.cpython-39.pyc and b/lvdm/models/__pycache__/ddpm3d.cpython-39.pyc differ
diff --git a/lvdm/models/__pycache__/utils_diffusion.cpython-39.pyc b/lvdm/models/__pycache__/utils_diffusion.cpython-39.pyc
index 797861db03fe3d01d425cce23a5794ab1cb57b04..2f14d044486d81a5f3c1e8e6000a5dd7eeb83200 100644
Binary files a/lvdm/models/__pycache__/utils_diffusion.cpython-39.pyc and b/lvdm/models/__pycache__/utils_diffusion.cpython-39.pyc differ
diff --git a/lvdm/models/samplers/__pycache__/ddim.cpython-39.pyc b/lvdm/models/samplers/__pycache__/ddim.cpython-39.pyc
index d21cd3e2e0cc26f0ddfced17f78135359590502f..9dfd449d835a3ae0090b2df8827ae556e112cf73 100644
Binary files a/lvdm/models/samplers/__pycache__/ddim.cpython-39.pyc and b/lvdm/models/samplers/__pycache__/ddim.cpython-39.pyc differ
diff --git a/lvdm/models/samplers/__pycache__/ddim_multiplecond.cpython-39.pyc b/lvdm/models/samplers/__pycache__/ddim_multiplecond.cpython-39.pyc
index db3c20dda87c515a8c881f1feca92c5ce40d7bbe..1576c6bae9ccfcc0d9be0c6279380e6d8db95e78 100644
Binary files a/lvdm/models/samplers/__pycache__/ddim_multiplecond.cpython-39.pyc and b/lvdm/models/samplers/__pycache__/ddim_multiplecond.cpython-39.pyc differ
diff --git a/lvdm/modules/__pycache__/attention.cpython-39.pyc b/lvdm/modules/__pycache__/attention.cpython-39.pyc
index 37cf4b94372832ceb72770c16ea269a5e015957f..f14120c94c45b857ff28a824db4511987576cd82 100644
Binary files a/lvdm/modules/__pycache__/attention.cpython-39.pyc and b/lvdm/modules/__pycache__/attention.cpython-39.pyc differ
diff --git a/lvdm/modules/encoders/__pycache__/condition.cpython-39.pyc b/lvdm/modules/encoders/__pycache__/condition.cpython-39.pyc
index 98c3e00eeeaa58c3d711c3794276ca09302672df..7cbe2dba11c84100f1712db6735529dcd7f19f77 100644
Binary files a/lvdm/modules/encoders/__pycache__/condition.cpython-39.pyc and b/lvdm/modules/encoders/__pycache__/condition.cpython-39.pyc differ
diff --git a/lvdm/modules/encoders/__pycache__/resampler.cpython-39.pyc b/lvdm/modules/encoders/__pycache__/resampler.cpython-39.pyc
index 65767ba4d5d71429c51b44e1bae61f777480d92f..6f8f273673eb305fc10f42946e764a73e51a81c4 100644
Binary files a/lvdm/modules/encoders/__pycache__/resampler.cpython-39.pyc and b/lvdm/modules/encoders/__pycache__/resampler.cpython-39.pyc differ
diff --git a/lvdm/modules/networks/__pycache__/ae_modules.cpython-39.pyc b/lvdm/modules/networks/__pycache__/ae_modules.cpython-39.pyc
index 485d9aa3706d9f985f8ae9c192a6f3ee34f5135a..671ed1dd1b63be83c5962863586a9fa0f969e2df 100644
Binary files a/lvdm/modules/networks/__pycache__/ae_modules.cpython-39.pyc and b/lvdm/modules/networks/__pycache__/ae_modules.cpython-39.pyc differ
diff --git a/lvdm/modules/networks/__pycache__/openaimodel3d.cpython-39.pyc b/lvdm/modules/networks/__pycache__/openaimodel3d.cpython-39.pyc
index dec7985a2cbb2c0410df2ccad8eaf5aafc6b19be..7db6b53d4059bcbba3783219217fa61904e9144f 100644
Binary files a/lvdm/modules/networks/__pycache__/openaimodel3d.cpython-39.pyc and b/lvdm/modules/networks/__pycache__/openaimodel3d.cpython-39.pyc differ
diff --git a/utils/__pycache__/diffusion_utils.cpython-39.pyc b/utils/__pycache__/diffusion_utils.cpython-39.pyc
index 5b36f44922bcecba9c5d9fca7b005c2a1e2cfdff..9d9853a4bcfb311832b4b01b1bbc50c4db9d4cbc 100644
Binary files a/utils/__pycache__/diffusion_utils.cpython-39.pyc and b/utils/__pycache__/diffusion_utils.cpython-39.pyc differ
diff --git a/utils/__pycache__/pvd_utils.cpython-39.pyc b/utils/__pycache__/pvd_utils.cpython-39.pyc
index bda72fb7a33ae2573f5614e6f67d4d0b37dff57f..f509e688b88e5c9e964931e0d055506666e3f63a 100644
Binary files a/utils/__pycache__/pvd_utils.cpython-39.pyc and b/utils/__pycache__/pvd_utils.cpython-39.pyc differ
diff --git a/viewcrafter.py b/viewcrafter.py
index 6b0a3d74b83aa898d87208b95d0bd85031d68233..5068642b8a43133ee2d0bb10bae9c8011873cdb7 100644
--- a/viewcrafter.py
+++ b/viewcrafter.py
@@ -32,7 +32,7 @@ class ViewCrafter:
self.opts = opts
self.device = opts.device
self.setup_dust3r()
- self.setup_diffusion()
+ self.setup_diffusion() #fixme
# initialize ref images, pcd
if not gradio:
self.images, self.img_ori = self.load_initial_images(image_dir=self.opts.image_dir)
@@ -160,11 +160,12 @@ class ViewCrafter:
render_results[-1] = self.img_ori
save_video(render_results, os.path.join(self.opts.save_dir, 'render0.mp4'))
- save_pointcloud_with_normals([imgs[-1]], [pcd[-1]], msk=None, save_path=os.path.join(self.opts.save_dir,'pcd0.ply') , mask_pc=False, reduce_pc=False)
- diffusion_results = self.run_diffusion(render_results)
- save_video((diffusion_results + 1.0) / 2.0, os.path.join(self.opts.save_dir, 'diffusion0.mp4'))
+ self.render_results = render_results
+ # save_pointcloud_with_normals([imgs[-1]], [pcd[-1]], msk=None, save_path=os.path.join(self.opts.save_dir,'pcd0.ply') , mask_pc=False, reduce_pc=False)
+ # diffusion_results = self.run_diffusion(render_results)
+ # save_video((diffusion_results + 1.0) / 2.0, os.path.join(self.opts.save_dir, 'diffusion0.mp4'))
- return diffusion_results
+ return None
def nvs_sparse_view(self,iter):
@@ -350,12 +351,10 @@ class ViewCrafter:
return images, img_ori
- def run_gradio(self,i2v_input_image, i2v_elevation, i2v_center_scale, i2v_d_phi, i2v_d_theta, i2v_d_r, i2v_steps, i2v_seed):
+ def run_traj(self,i2v_input_image, i2v_elevation, i2v_center_scale, i2v_d_phi, i2v_d_theta, i2v_d_r):
self.opts.elevation = float(i2v_elevation)
self.opts.center_scale = float(i2v_center_scale)
- self.opts.ddim_steps = i2v_steps
self.gradio_traj = [float(i) for i in i2v_d_phi.split()],[float(i) for i in i2v_d_theta.split()],[float(i) for i in i2v_d_r.split()]
- seed_everything(i2v_seed)
transform = transforms.Compose([
transforms.Resize(576),
transforms.CenterCrop((576,1024)),
@@ -375,8 +374,13 @@ class ViewCrafter:
# self.images, self.img_ori = self.load_initial_images(image_dir=i2v_input_image)
self.run_dust3r(input_images=self.images)
self.nvs_single_view(gradio=True)
-
traj_dir = os.path.join(self.opts.save_dir, "viz_traj.mp4")
+ return traj_dir
+
+ def run_gen(self,i2v_steps, i2v_seed):
+ self.opts.ddim_steps = i2v_steps
+ seed_everything(i2v_seed)
gen_dir = os.path.join(self.opts.save_dir, "diffusion0.mp4")
-
- return traj_dir, gen_dir
\ No newline at end of file
+ diffusion_results = self.run_diffusion(self.render_results)
+ save_video((diffusion_results + 1.0) / 2.0, os.path.join(self.opts.save_dir, 'diffusion0.mp4'))
+ return gen_dir
\ No newline at end of file