update

log.txt

@@ -0,0 +1,297 @@
+Logging to ./
+creating model and diffusion...
+creating 3DAE...
+length of vit_decoder.blocks: 24
+init pos_embed with sincos
+length of vit_decoder.blocks: 24
+ignore dim_up_mlp:  True
+AE(
+  (encoder): MVEncoderGSDynamicInp(
+    (conv_in): Conv2d(10, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+    (down): ModuleList(
+      (0): Module(
+        (block): ModuleList(
+          (0): ResnetBlock(
+            (norm1): GroupNorm(32, 64, eps=1e-06, affine=True)
+            (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+            (norm2): GroupNorm(32, 64, eps=1e-06, affine=True)
+            (dropout): Dropout(p=0.0, inplace=False)
+            (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+          )
+        )
+        (attn): ModuleList()
+        (downsample): Downsample(
+          (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(2, 2))
+        )
+      )
+      (1): Module(
+        (block): ModuleList(
+          (0): ResnetBlock(
+            (norm1): GroupNorm(32, 64, eps=1e-06, affine=True)
+            (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+            (norm2): GroupNorm(32, 128, eps=1e-06, affine=True)
+            (dropout): Dropout(p=0.0, inplace=False)
+            (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+            (nin_shortcut): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1))
+          )
+        )
+        (attn): ModuleList()
+        (downsample): Downsample(
+          (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(2, 2))
+        )
+      )
+      (2): Module(
+        (block): ModuleList(
+          (0): ResnetBlock(
+            (norm1): GroupNorm(32, 128, eps=1e-06, affine=True)
+            (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+            (norm2): GroupNorm(32, 256, eps=1e-06, affine=True)
+            (dropout): Dropout(p=0.0, inplace=False)
+            (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+            (nin_shortcut): Conv2d(128, 256, kernel_size=(1, 1), stride=(1, 1))
+          )
+        )
+        (attn): ModuleList()
+        (downsample): Downsample(
+          (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2))
+        )
+      )
+      (3): Module(
+        (block): ModuleList(
+          (0): ResnetBlock(
+            (norm1): GroupNorm(32, 256, eps=1e-06, affine=True)
+            (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+            (norm2): GroupNorm(32, 256, eps=1e-06, affine=True)
+            (dropout): Dropout(p=0.0, inplace=False)
+            (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+          )
+        )
+        (attn): ModuleList()
+      )
+    )
+    (mid): Module(
+      (block_1): ResnetBlock(
+        (norm1): GroupNorm(32, 256, eps=1e-06, affine=True)
+        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+        (norm2): GroupNorm(32, 256, eps=1e-06, affine=True)
+        (dropout): Dropout(p=0.0, inplace=False)
+        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+      )
+      (attn_1): SpatialTransformer3D(
+        (norm): GroupNorm(32, 256, eps=1e-06, affine=True)
+        (proj_in): Conv2d(256, 512, kernel_size=(1, 1), stride=(1, 1))
+        (transformer_blocks): ModuleList(
+          (0): BasicTransformerBlock3D(
+            (attn1): MemoryEfficientCrossAttention(
+              (to_q): Linear(in_features=512, out_features=512, bias=False)
+              (to_k): Linear(in_features=512, out_features=512, bias=False)
+              (q_norm): Identity()
+              (k_norm): Identity()
+              (to_v): Linear(in_features=512, out_features=512, bias=False)
+              (to_out): Sequential(
+                (0): Linear(in_features=512, out_features=512, bias=True)
+                (1): Dropout(p=0.0, inplace=False)
+              )
+            )
+            (ff): FeedForward(
+              (net): Sequential(
+                (0): GEGLU(
+                  (proj): Linear(in_features=512, out_features=4096, bias=True)
+                )
+                (1): Dropout(p=0.0, inplace=False)
+                (2): Linear(in_features=2048, out_features=512, bias=True)
+              )
+            )
+            (attn2): MemoryEfficientCrossAttention(
+              (to_q): Linear(in_features=512, out_features=512, bias=False)
+              (to_k): Linear(in_features=512, out_features=512, bias=False)
+              (q_norm): Identity()
+              (k_norm): Identity()
+              (to_v): Linear(in_features=512, out_features=512, bias=False)
+              (to_out): Sequential(
+                (0): Linear(in_features=512, out_features=512, bias=True)
+                (1): Dropout(p=0.0, inplace=False)
+              )
+            )
+            (norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+            (norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+            (norm3): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+          )
+        )
+        (proj_out): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1))
+      )
+      (block_2): ResnetBlock(
+        (norm1): GroupNorm(32, 256, eps=1e-06, affine=True)
+        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+        (norm2): GroupNorm(32, 256, eps=1e-06, affine=True)
+        (dropout): Dropout(p=0.0, inplace=False)
+        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+      )
+    )
+    (norm_out): GroupNorm(32, 256, eps=1e-06, affine=True)
+    (conv_out): Conv2d(256, 24, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+  )
+  (decoder): RodinSR_256_fusionv6_ConvQuant_liteSR_dinoInit3DAttn_SD_B_3L_C_withrollout_withSD_D_ditDecoder(
+    (superresolution): ModuleDict(
+      (ldm_upsample): PatchEmbedTriplane(
+        (proj): Conv2d(12, 3072, kernel_size=(2, 2), stride=(2, 2), groups=3)
+        (norm): Identity()
+      )
+      (quant_conv): Conv2d(24, 24, kernel_size=(1, 1), stride=(1, 1), groups=3)
+      (conv_sr): Decoder(
+        (conv_in): Conv2d(1024, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+        (mid): Module(
+          (block_1): ResnetBlock(
+            (norm1): GroupNorm(32, 128, eps=1e-06, affine=True)
+            (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+            (norm2): GroupNorm(32, 128, eps=1e-06, affine=True)
+            (dropout): Dropout(p=0.0, inplace=False)
+            (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+          )
+          (attn_1): MemoryEfficientAttnBlock(
+            (norm): GroupNorm(32, 128, eps=1e-06, affine=True)
+            (q): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))
+            (k): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))
+            (v): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))
+            (proj_out): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))
+          )
+          (block_2): ResnetBlock(
+            (norm1): GroupNorm(32, 128, eps=1e-06, affine=True)
+            (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+            (norm2): GroupNorm(32, 128, eps=1e-06, affine=True)
+            (dropout): Dropout(p=0.0, inplace=False)
+            (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+          )
+        )
+        (up): ModuleList(
+          (0): Module(
+            (block): ModuleList(
+              (0): ResnetBlock(
+                (norm1): GroupNorm(32, 64, eps=1e-06, affine=True)
+                (conv1): Conv2d(64, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+                (norm2): GroupNorm(32, 32, eps=1e-06, affine=True)
+                (dropout): Dropout(p=0.0, inplace=False)
+                (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+                (nin_shortcut): Conv2d(64, 32, kernel_size=(1, 1), stride=(1, 1))
+              )
+              (1): ResnetBlock(
+                (norm1): GroupNorm(32, 32, eps=1e-06, affine=True)
+                (conv1): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+                (norm2): GroupNorm(32, 32, eps=1e-06, affine=True)
+                (dropout): Dropout(p=0.0, inplace=False)
+                (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+              )
+            )
+            (attn): ModuleList()
+          )
+          (1): Module(
+            (block): ModuleList(
+              (0-1): 2 x ResnetBlock(
+                (norm1): GroupNorm(32, 64, eps=1e-06, affine=True)
+                (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+                (norm2): GroupNorm(32, 64, eps=1e-06, affine=True)
+                (dropout): Dropout(p=0.0, inplace=False)
+                (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+              )
+            )
+            (attn): ModuleList()
+            (upsample): Upsample(
+              (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+            )
+          )
+          (2): Module(
+            (block): ModuleList(
+              (0): ResnetBlock(
+                (norm1): GroupNorm(32, 128, eps=1e-06, affine=True)
+                (conv1): Conv2d(128, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+                (norm2): GroupNorm(32, 64, eps=1e-06, affine=True)
+                (dropout): Dropout(p=0.0, inplace=False)
+                (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+                (nin_shortcut): Conv2d(128, 64, kernel_size=(1, 1), stride=(1, 1))
+              )
+              (1): ResnetBlock(
+                (norm1): GroupNorm(32, 64, eps=1e-06, affine=True)
+                (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+                (norm2): GroupNorm(32, 64, eps=1e-06, affine=True)
+                (dropout): Dropout(p=0.0, inplace=False)
+                (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+              )
+            )
+            (attn): ModuleList()
+            (upsample): Upsample(
+              (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+            )
+          )
+          (3): Module(
+            (block): ModuleList(
+              (0-1): 2 x ResnetBlock(
+                (norm1): GroupNorm(32, 128, eps=1e-06, affine=True)
+                (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+                (norm2): GroupNorm(32, 128, eps=1e-06, affine=True)
+                (dropout): Dropout(p=0.0, inplace=False)
+                (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+              )
+            )
+            (attn): ModuleList()
+            (upsample): Upsample(
+              (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+            )
+          )
+        )
+        (norm_out): GroupNorm(32, 32, eps=1e-06, affine=True)
+        (conv_out): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+      )
+    )
+    (vit_decoder): DiT2(
+      (blocks): ModuleList(
+        (0-23): 24 x DiTBlock2(
+          (norm1): LayerNorm((1024,), eps=1e-06, elementwise_affine=False)
+          (norm2): LayerNorm((1024,), eps=1e-06, elementwise_affine=False)
+          (attn): MemEffAttention(
+            (qkv): Linear(in_features=1024, out_features=3072, bias=True)
+            (attn_drop): Dropout(p=0.0, inplace=False)
+            (proj): Linear(in_features=1024, out_features=1024, bias=True)
+            (proj_drop): Dropout(p=0.0, inplace=False)
+            (q_norm): Identity()
+            (k_norm): Identity()
+          )
+          (mlp): FusedMLP(
+            (mlp): Sequential(
+              (0): Linear(in_features=1024, out_features=4096, bias=False)
+              (1): FusedDropoutBias(
+                (activation_pytorch): GELU(approximate='none')
+              )
+              (2): Linear(in_features=4096, out_features=1024, bias=False)
+              (3): FusedDropoutBias(
+                (activation_pytorch): Identity()
+              )
+            )
+          )
+          (adaLN_modulation): Sequential(
+            (0): SiLU()
+            (1): Linear(in_features=1024, out_features=6144, bias=True)
+          )
+        )
+      )
+    )
+    (triplane_decoder): Triplane(
+      (renderer): ImportanceRenderer(
+        (ray_marcher): MipRayMarcher2()
+      )
+      (ray_sampler): PatchRaySampler()
+      (decoder): OSGDecoder(
+        (net): Sequential(
+          (0): FullyConnectedLayer(in_features=32, out_features=64, activation=linear)
+          (1): Softplus(beta=1.0, threshold=20.0)
+          (2): FullyConnectedLayer(in_features=64, out_features=4, activation=linear)
+        )
+      )
+    )
+    (decoder_pred): None
+  )
+)
+create dataset
+joint_denoise_rec_model enables AMP to accelerate training
+mark joint_denoise_rec_model loading 
+loading model from huggingface: yslan/LN3Diff/checkpoints/objaverse/objaverse-dit/i23d/model_joint_denoise_rec_model2990000.safetensors...
+mark joint_denoise_rec_model loading finished

nsr/lsgm/flow_matching_trainer.py

@@ -693,9 +693,10 @@ class FlowMatchingEngine(TrainLoop3DDiffusionLSGM_crossattn):
         **kwargs,
     ):
 
-        # output_model, output_video = './logs/LSGM/inference/Objaverse/i23d/dit-L2/gradio_app/mesh/cfg=4.0_sample-0.ply', './logs/LSGM/inference/Objaverse/i23d/dit-L2/gradio_app/triplane_cfg=4.0_sample-0.mp4'
+        output_model= './logs/LSGM/inference/Objaverse/i23d/dit-L2/gradio_app/mesh/cfg=4.0_sample-0.ply'
+        output_video =  './logs/LSGM/inference/Objaverse/i23d/dit-L2/gradio_app/triplane_cfg=4.0_sample-0.mp4'
 
-        # return output_model, output_video
+        return output_video, output_model
 
         camera = th.load('assets/objv_eval_pose.pt', map_location=dist_util.dev())[:]
         inp_img = th.from_numpy(inp_img).permute(2,0,1).unsqueeze(0) / 127.5 - 1 # to [-1,1]

runs/events.out.tfevents.1724070673.yslan-slab.2214140.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:781470717192cf2eb69563802a7f887853902e1eac7367d15c48f186ac0fbdfe
+size 88

runs/events.out.tfevents.1724070845.yslan-slab.2216339.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2826c0d11fbf9be35937c5aefda343633ab8367dc4f903ffbfd13257448a849b
+size 88

runs/events.out.tfevents.1724070935.yslan-slab.2217340.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ca1a10b2f9cc18e1692678c345777d60670424612acdc655bc38077e3f2c279
+size 88

runs/events.out.tfevents.1724071056.yslan-slab.2218899.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b77a71e88b2cf377b7174d52544cadeb126fc21bdf4a2fc47dbfed7966c72d50
+size 88

runs/events.out.tfevents.1724071116.yslan-slab.2220227.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c34ee13e40fe78faf1e5822d6e22936f0c2a394aab8080426eede96617cb2b9
+size 88