Update config.json

config.json

@@ -1,26 +1,224 @@
-{
-  "model_config": {
-    "vlm_path": "microsoft/Florence-2-large",
-    "freeze_florence": false,
-    "freeze_vision_tower": false,
-    "vlm_prompt_style": "default",
-    "token_dropout": 0.1,
-    "multistep": 10,
-    "num_sampling_steps": 4,
-    "lowdim_obs_dim": 7,
-    "action_dim": 7,
-    "act_window_size": 10,
-    "use_second_view": true,
-    "second_view_key": "image_wrist",
-    "action_type_adaln": true,
-    "use_causal_attention": true,
-    "use_cross_attn": true,
-    "sampling_type": "uniform",
-    "dit_dim": 1024,
-    "n_heads": 16,
-    "n_layers": 18,
-    "use_rope": true,
-    "query_seq_len": 100,
-    "rope_theta": 32.0
-  }
-}
+callbacks:
+  ema:
+    _target_: flower.callbacks.ema.EMA
+    decay: 0.999
+    start_step: 0
+    save_ema_weights_in_callback_state: true
+    evaluate_ema_weights_instead: true
+    power: 0.6666666666666666
+    inv_gamma: 1.0
+    min_value: 0.0
+    max_value: 0.9999
+  rollout_lh:
+    _target_: flower.rollout.libero_rollout.RolloutLibero
+    _recursive_: false
+    env_cfg:
+      _target_: flower.wrappers.hulc_wrapper.HulcWrapper
+    skip_epochs: ${rollout_lh_skip_epochs}
+    benchmark_name: ${libero_benchmark}
+    rollout_freq: 10
+    num_videos: 0
+    num_sequences: 20
+    max_steps: 520
+    empty_cache: false
+    debug: false
+    n_eval: 50
+    num_procs: 10
+    use_mp: false
+    task_embedding_format: clip
+    device: ${device}
+  checkpoint:
+    _target_: pytorch_lightning.callbacks.ModelCheckpoint
+    save_top_k: 1
+    verbose: true
+    monitor: eval_lh/avg_seq_len
+    mode: max
+    dirpath: saved_models
+    filename: '{epoch:02d}_{eval_lh/avg_seq_len:.2f}'
+    every_n_epochs: ${callbacks.rollout_lh.rollout_freq}
+datamodule:
+  transforms:
+    train:
+      rgb_static:
+      - _target_: torchvision.transforms.Resize
+        size: 112
+        antialias: true
+      - _target_: flower.utils.transforms.RandomShiftsAug
+        pad: 10
+      - _target_: flower.utils.transforms.ScaleImageTensor
+      - _target_: torchvision.transforms.Normalize
+        mean:
+        - 0.48145466
+        - 0.4578275
+        - 0.40821073
+        std:
+        - 0.26862954
+        - 0.26130258
+        - 0.27577711
+      rgb_gripper:
+      - _target_: torchvision.transforms.Resize
+        size: 112
+        antialias: true
+      - _target_: flower.utils.transforms.RandomShiftsAug
+        pad: 4
+      - _target_: flower.utils.transforms.ScaleImageTensor
+      - _target_: torchvision.transforms.Normalize
+        mean:
+        - 0.48145466
+        - 0.4578275
+        - 0.40821073
+        std:
+        - 0.26862954
+        - 0.26130258
+        - 0.27577711
+    val:
+      rgb_static:
+      - _target_: torchvision.transforms.Resize
+        size: 112
+        antialias: true
+      - _target_: flower.utils.transforms.ScaleImageTensor
+      - _target_: torchvision.transforms.Normalize
+        mean:
+        - 0.48145466
+        - 0.4578275
+        - 0.40821073
+        std:
+        - 0.26862954
+        - 0.26130258
+        - 0.27577711
+      rgb_gripper:
+      - _target_: torchvision.transforms.Resize
+        size: 112
+        antialias: true
+      - _target_: flower.utils.transforms.ScaleImageTensor
+      - _target_: torchvision.transforms.Normalize
+        mean:
+        - 0.48145466
+        - 0.4578275
+        - 0.40821073
+        std:
+        - 0.26862954
+        - 0.26130258
+        - 0.27577711
+  _target_: flower.datasets.libero_data_module.LiberoDataModule
+  _recursive_: false
+  root_data_dir: ${root_data_dir}
+  action_space: 7
+  shuffle_val: false
+  benchmark_name: ${libero_benchmark}
+  observation_space:
+    rgb_obs:
+    - agentview_rgb
+    - eye_in_hand_rgb
+    depth_obs: []
+    state_obs:
+    - gripper_states
+    - joint_states
+    actions:
+    - rel_actions
+    language:
+    - language
+  proprioception_dims: None
+  datasets:
+    lang_dataset:
+      _target_: flower.datasets.libero_dataset.LiberoMultitaskDataset
+      key: lang
+      benchmark_name: ${libero_benchmark}
+      batch_size: ${batch_size}
+      proprio_state: ${datamodule.proprioception_dims}
+      obs_space: ${datamodule.observation_space}
+      num_workers: ${num_workers}
+      action_seq_len: ${act_seq_len}
+      obs_seq_len: ${obs_seq_len}
+      split_ratio: 0.0
+model:
+  _target_: flower.models.flower.FLOWERVLA
+  _recursive_: false
+  vlm_path: microsoft/Florence-2-large
+  freeze_florence: false
+  freeze_vision_tower: false
+  vlm_prompt_style: default
+  token_dropout: 0.1
+  multistep: ${multistep}
+  num_sampling_steps: 4
+  lowdim_obs_dim: 7
+  action_dim: 7
+  act_window_size: 10
+  load_pretrained: true
+  pretrained_model_path: /home/hk-project-sustainebot/ft4740/code/flower_vla_policy/logs/runs/2025-02-05/10-17-02/360000_model_weights.pt
+  use_second_view: true
+  second_view_key: image_wrist
+  action_type_adaln: true
+  use_causal_attention: true
+  use_cross_attn: true
+  use_adaln_cond: false
+  use_readout_token: false
+  use_proprio: false
+  return_act_chunk: false
+  sampling_type: uniform
+  dit_dim: 1024
+  n_heads: 16
+  n_layers: 18
+  attn_pdrop: 0.1
+  resid_pdrop: 0.1
+  mlp_pdrop: 0.1
+  use_rope: true
+  use_nope: false
+  query_seq_len: 100
+  rope_theta: 32.0
+  optimizer_type: adamw
+  optimizer:
+    _target_: torch.optim.AdamW
+    transformer_weight_decay: 0.05
+    learning_rate: 2.0e-05
+    betas:
+    - 0.9
+    - 0.95
+  lr_scheduler:
+    lr_scheduler:
+      init_lr: 2.0e-05
+      init_lr_scale: 0.1
+      final_lr_scale: 0.5
+      total_steps: 50000
+      phase_ratio: (0.05, 0.1, 0.85)
+      lr: 2.0e-05
+root_data_dir: /home/yagmurlu/code/MoDE_Calvin/dataset/task_ABC_D
+lang_folder: lang_clip_resnet50
+log_dir: ./logs
+slurm: false
+seed: 42
+device: cuda
+batch_size: 8
+devices: 4
+goal_window_size: 1
+act_dim: 7
+proprio_dims: 9
+obs_dim: 512
+goal_dim: 512
+obs_seq_len: 1
+act_seq_len: 10
+multistep: ${act_seq_len}
+p_last_state: 0
+max_epochs: 50
+rollout_lh_skip_epochs: 19
+num_workers: 1
+benchmark_name: ${libero_benchmark}
+libero_benchmark: libero_90
+trainer:
+  devices: ${devices}
+  precision: bf16-mixed
+  max_epochs: ${max_epochs}
+  sync_batchnorm: true
+  accelerator: gpu
+  strategy: ddp
+  limit_train_batches: 1000
+  limit_val_batches: 4
+logger:
+  _target_: pytorch_lightning.loggers.WandbLogger
+  save_dir: .
+  name: logger
+  group: mode
+  log_model: false
+  project: ${libero_benchmark}
+  entity: bennoq
+  id: ???