Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+wandb/latest-run/run-jcj67gg8.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241105_193102-jcj67gg8/run-jcj67gg8.wandb filter=lfs diff=lfs merge=lfs -text

checkpoints/step-009999-epoch-00-loss=0.7280.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcb0567c48a8af51b96e8b73b794de80964badc806defd7badc77007782a5a0a
+size 2630986501

checkpoints/step-019999-epoch-01-loss=1.4845.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71573cb46a219059c298c41f8facc10271b95e0a4cb3c1e0e8c39ac3a66079b8
+size 2630986501

checkpoints/step-020792-epoch-01-loss=0.5268.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d86525b8d21865b2e83d4a403ee9d1b641579bb41259111d7e3cc7f7ed46564a
+size 2630986501

config.json

@@ -0,0 +1,61 @@
+{
+  "dataset": {
+    "align_stage_components": [
+      "download/llava-laion-cc-sbu-558k/chat.json",
+      "download/llava-laion-cc-sbu-558k"
+    ],
+    "dataset_id": "llava-v15",
+    "dataset_root_dir": "/hai/scratch/belkhale/datasets/prismatic-vlms",
+    "finetune_stage_components": [
+      "download/llava-v1.5-instruct/llava_v1_5_mix665k.json",
+      "download/llava-v1.5-instruct"
+    ],
+    "type": "llava-v15"
+  },
+  "hf_token": ".hf_token",
+  "model": {
+    "align_epochs": 1,
+    "align_global_batch_size": 96,
+    "align_learning_rate": 0.001,
+    "align_lr_scheduler_type": "linear-warmup+cosine-decay",
+    "align_max_grad_norm": 1.0,
+    "align_max_steps": null,
+    "align_per_device_batch_size": 16,
+    "align_save_every_n_steps": 10000,
+    "align_train_strategy": "fsdp-shard-grad-op",
+    "align_warmup_ratio": 0.03,
+    "align_weight_decay": 0.0,
+    "arch_specifier": "no-align+fused-gelu-mlp",
+    "enable_gradient_checkpointing": true,
+    "enable_mixed_precision_training": true,
+    "finetune_epochs": 2,
+    "finetune_global_batch_size": 64,
+    "finetune_learning_rate": 2e-05,
+    "finetune_lr_scheduler_type": "linear-warmup+cosine-decay",
+    "finetune_max_grad_norm": 1.0,
+    "finetune_max_steps": null,
+    "finetune_per_device_batch_size": 4,
+    "finetune_save_every_n_steps": 10000,
+    "finetune_train_strategy": "fsdp-full-shard",
+    "finetune_warmup_ratio": 0.03,
+    "finetune_weight_decay": 0.1,
+    "image_resize_strategy": "resize-naive",
+    "llm_backbone_id": "qwen25-0_5b-extra",
+    "llm_max_length": 32768,
+    "model_id": "prism-qwen25-extra-dinosiglip-224px+0_5b",
+    "reduce_in_full_precision": false,
+    "type": "prism-qwen25-extra-dinosiglip-224px+0_5b",
+    "vision_backbone_id": "dinosiglip-vit-so-224px"
+  },
+  "pretrained_checkpoint": null,
+  "run_id": "prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7",
+  "run_root_dir": "runs",
+  "seed": 7,
+  "stage": "finetune",
+  "trackers": [
+    "jsonl",
+    "wandb"
+  ],
+  "wandb_entity": null,
+  "wandb_project": "prismatic"
+}

config.yaml

@@ -0,0 +1,54 @@
+dataset:
+  align_stage_components:
+  - download/llava-laion-cc-sbu-558k/chat.json
+  - download/llava-laion-cc-sbu-558k
+  dataset_id: llava-v15
+  dataset_root_dir: /hai/scratch/belkhale/datasets/prismatic-vlms
+  finetune_stage_components:
+  - download/llava-v1.5-instruct/llava_v1_5_mix665k.json
+  - download/llava-v1.5-instruct
+  type: llava-v15
+hf_token: .hf_token
+model:
+  align_epochs: 1
+  align_global_batch_size: 96
+  align_learning_rate: 0.001
+  align_lr_scheduler_type: linear-warmup+cosine-decay
+  align_max_grad_norm: 1.0
+  align_max_steps: null
+  align_per_device_batch_size: 16
+  align_save_every_n_steps: 10000
+  align_train_strategy: fsdp-shard-grad-op
+  align_warmup_ratio: 0.03
+  align_weight_decay: 0.0
+  arch_specifier: no-align+fused-gelu-mlp
+  enable_gradient_checkpointing: true
+  enable_mixed_precision_training: true
+  finetune_epochs: 2
+  finetune_global_batch_size: 64
+  finetune_learning_rate: 2.0e-05
+  finetune_lr_scheduler_type: linear-warmup+cosine-decay
+  finetune_max_grad_norm: 1.0
+  finetune_max_steps: null
+  finetune_per_device_batch_size: 4
+  finetune_save_every_n_steps: 10000
+  finetune_train_strategy: fsdp-full-shard
+  finetune_warmup_ratio: 0.03
+  finetune_weight_decay: 0.1
+  image_resize_strategy: resize-naive
+  llm_backbone_id: qwen25-0_5b-extra
+  llm_max_length: 32768
+  model_id: prism-qwen25-extra-dinosiglip-224px+0_5b
+  reduce_in_full_precision: false
+  type: prism-qwen25-extra-dinosiglip-224px+0_5b
+  vision_backbone_id: dinosiglip-vit-so-224px
+pretrained_checkpoint: null
+run_id: prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7
+run_root_dir: runs
+seed: 7
+stage: finetune
+trackers:
+- jsonl
+- wandb
+wandb_entity: null
+wandb_project: prismatic

run-metrics.jsonl

@@ -0,0 +1 @@
+{"hparams": {"dataset": {"align_stage_components": ["download/llava-laion-cc-sbu-558k/chat.json", "download/llava-laion-cc-sbu-558k"], "dataset_id": "llava-v15", "dataset_root_dir": "/hai/scratch/belkhale/datasets/prismatic-vlms", "finetune_stage_components": ["download/llava-v1.5-instruct/llava_v1_5_mix665k.json", "download/llava-v1.5-instruct"], "type": "llava-v15"}, "hf_token": ".hf_token", "model": {"align_epochs": 1, "align_global_batch_size": 96, "align_learning_rate": 0.001, "align_lr_scheduler_type": "linear-warmup+cosine-decay", "align_max_grad_norm": 1.0, "align_max_steps": null, "align_per_device_batch_size": 16, "align_save_every_n_steps": 10000, "align_train_strategy": "fsdp-shard-grad-op", "align_warmup_ratio": 0.03, "align_weight_decay": 0.0, "arch_specifier": "no-align+fused-gelu-mlp", "enable_gradient_checkpointing": true, "enable_mixed_precision_training": true, "finetune_epochs": 2, "finetune_global_batch_size": 64, "finetune_learning_rate": 2e-05, "finetune_lr_scheduler_type": "linear-warmup+cosine-decay", "finetune_max_grad_norm": 1.0, "finetune_max_steps": null, "finetune_per_device_batch_size": 4, "finetune_save_every_n_steps": 10000, "finetune_train_strategy": "fsdp-full-shard", "finetune_warmup_ratio": 0.03, "finetune_weight_decay": 0.1, "image_resize_strategy": "resize-naive", "llm_backbone_id": "qwen25-0_5b-extra", "llm_max_length": 32768, "model_id": "prism-qwen25-extra-dinosiglip-224px+0_5b", "reduce_in_full_precision": false, "type": "prism-qwen25-extra-dinosiglip-224px+0_5b", "vision_backbone_id": "dinosiglip-vit-so-224px"}, "pretrained_checkpoint": null, "run_id": "prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7", "run_root_dir": "runs", "seed": 7, "stage": "finetune", "trackers": ["jsonl", "wandb"], "wandb_entity": null, "wandb_project": "prismatic"}, "run_id": "prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7"}

wandb/debug-internal.log

@@ -0,0 +1,17 @@
+{"time":"2024-11-05T19:31:02.519998294-08:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-05T19:31:02.520021589-08:00","level":"INFO","msg":"created symlink","path":"runs/prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7/wandb/run-20241105_193102-jcj67gg8/logs/debug-core.log"}
+{"time":"2024-11-05T19:31:02.738801523-08:00","level":"INFO","msg":"created new stream","id":"jcj67gg8"}
+{"time":"2024-11-05T19:31:02.738835256-08:00","level":"INFO","msg":"stream: started","id":"jcj67gg8"}
+{"time":"2024-11-05T19:31:02.738905513-08:00","level":"INFO","msg":"sender: started","stream_id":"jcj67gg8"}
+{"time":"2024-11-05T19:31:02.738892436-08:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"jcj67gg8"}}
+{"time":"2024-11-05T19:31:02.738902832-08:00","level":"INFO","msg":"handler: started","stream_id":{"value":"jcj67gg8"}}
+{"time":"2024-11-05T19:31:03.056521545-08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-05T23:38:31.946246118-08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-05T23:38:32.023973784-08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-05T23:38:32.922341453-08:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"saving job artifact","runtime_seconds":0.775009129}],"total_operations":1}}
+{"time":"2024-11-05T23:38:33.390878425-08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-05T23:38:34.555591357-08:00","level":"INFO","msg":"stream: closing","id":"jcj67gg8"}
+{"time":"2024-11-05T23:38:34.555667186-08:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"jcj67gg8"}}
+{"time":"2024-11-05T23:38:34.555706801-08:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"jcj67gg8"}}
+{"time":"2024-11-05T23:38:34.555872-08:00","level":"INFO","msg":"sender: closed","stream_id":"jcj67gg8"}
+{"time":"2024-11-05T23:38:34.558526671-08:00","level":"INFO","msg":"stream: closed","id":"jcj67gg8"}

wandb/debug.log

@@ -0,0 +1,32 @@
+2024-11-05 19:31:02,511 INFO    MainThread:2188020 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-05 19:31:02,511 INFO    MainThread:2188020 [wandb_setup.py:_flush():79] Configure stats pid to 2188020
+2024-11-05 19:31:02,511 INFO    MainThread:2188020 [wandb_setup.py:_flush():79] Loading settings from /hai/scratch/belkhale/.config/wandb/settings
+2024-11-05 19:31:02,511 INFO    MainThread:2188020 [wandb_setup.py:_flush():79] Loading settings from /hai/scratch/belkhale/openvla-mini/wandb/settings
+2024-11-05 19:31:02,511 INFO    MainThread:2188020 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'_service_wait': '300'}
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/hai/scratch/belkhale/openvla-mini/scripts/pretrain.py', 'program': '/hai/scratch/belkhale/openvla-mini/scripts/pretrain.py'}
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_init.py:_log_setup():534] Logging user logs to runs/prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7/wandb/run-20241105_193102-jcj67gg8/logs/debug.log
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_init.py:_log_setup():535] Logging internal logs to runs/prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7/wandb/run-20241105_193102-jcj67gg8/logs/debug-internal.log
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_init.py:init():621] calling init triggers
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {'model': {'type': 'prism-qwen25-extra-dinosiglip-224px+0_5b', 'model_id': 'prism-qwen25-extra-dinosiglip-224px+0_5b', 'arch_specifier': 'no-align+fused-gelu-mlp', 'vision_backbone_id': 'dinosiglip-vit-so-224px', 'llm_backbone_id': 'qwen25-0_5b-extra', 'image_resize_strategy': 'resize-naive', 'llm_max_length': 32768, 'align_epochs': 1, 'align_max_steps': None, 'align_save_every_n_steps': 10000, 'align_global_batch_size': 96, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 2, 'finetune_max_steps': None, 'finetune_save_every_n_steps': 10000, 'finetune_global_batch_size': 64, 'finetune_per_device_batch_size': 4, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'llava-v15', 'dataset_id': 'llava-v15', 'align_stage_components': ['download/llava-laion-cc-sbu-558k/chat.json', 'download/llava-laion-cc-sbu-558k'], 'finetune_stage_components': ['download/llava-v1.5-instruct/llava_v1_5_mix665k.json', 'download/llava-v1.5-instruct'], 'dataset_root_dir': '/hai/scratch/belkhale/datasets/prismatic-vlms'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'hf_token': '.hf_token', 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'prismatic', 'wandb_entity': None}
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_init.py:init():671] starting backend
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_init.py:init():675] sending inform_init request
+2024-11-05 19:31:02,513 INFO    MainThread:2188020 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-05 19:31:02,513 INFO    MainThread:2188020 [wandb_init.py:init():688] backend started and connected
+2024-11-05 19:31:02,515 INFO    MainThread:2188020 [wandb_init.py:init():783] updated telemetry
+2024-11-05 19:31:02,573 INFO    MainThread:2188020 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-05 19:31:03,050 INFO    MainThread:2188020 [wandb_init.py:init():867] starting run threads in backend
+2024-11-05 19:31:03,226 INFO    MainThread:2188020 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-05 19:31:03,227 INFO    MainThread:2188020 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-05 19:31:03,227 INFO    MainThread:2188020 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-05 19:31:03,227 INFO    MainThread:2188020 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-05 19:31:03,230 INFO    MainThread:2188020 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-05 23:38:31,920 INFO    MainThread:2188020 [wandb_run.py:_finish():2158] finishing run belkhale/prismatic/jcj67gg8
+2024-11-05 23:38:31,920 INFO    MainThread:2188020 [wandb_run.py:_atexit_cleanup():2426] got exitcode: 0
+2024-11-05 23:38:31,921 INFO    MainThread:2188020 [wandb_run.py:_restore():2408] restore
+2024-11-05 23:38:31,921 INFO    MainThread:2188020 [wandb_run.py:_restore():2414] restore done
+2024-11-05 23:38:34,516 INFO    MainThread:2188020 [wandb_run.py:_footer_history_summary_info():3975] rendering history
+2024-11-05 23:38:34,517 INFO    MainThread:2188020 [wandb_run.py:_footer_history_summary_info():4007] rendering summary
+2024-11-05 23:38:34,534 INFO    MainThread:2188020 [wandb_run.py:_footer_sync_info():3934] logging synced files

wandb/latest-run/files/config.yaml

@@ -0,0 +1,105 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.10.15
+        t:
+            "1":
+                - 1
+                - 2
+                - 3
+                - 11
+                - 41
+                - 49
+                - 55
+                - 63
+                - 71
+            "2":
+                - 1
+                - 2
+                - 3
+                - 11
+                - 41
+                - 49
+                - 55
+                - 63
+                - 71
+            "3":
+                - 2
+                - 13
+                - 16
+                - 23
+                - 55
+                - 61
+            "4": 3.10.15
+            "5": 0.18.5
+            "6": 4.40.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+dataset:
+    value:
+        align_stage_components:
+            - download/llava-laion-cc-sbu-558k/chat.json
+            - download/llava-laion-cc-sbu-558k
+        dataset_id: llava-v15
+        dataset_root_dir: /hai/scratch/belkhale/datasets/prismatic-vlms
+        finetune_stage_components:
+            - download/llava-v1.5-instruct/llava_v1_5_mix665k.json
+            - download/llava-v1.5-instruct
+        type: llava-v15
+hf_token:
+    value: .hf_token
+model:
+    value:
+        align_epochs: 1
+        align_global_batch_size: 96
+        align_learning_rate: 0.001
+        align_lr_scheduler_type: linear-warmup+cosine-decay
+        align_max_grad_norm: 1
+        align_max_steps: null
+        align_per_device_batch_size: 16
+        align_save_every_n_steps: 10000
+        align_train_strategy: fsdp-shard-grad-op
+        align_warmup_ratio: 0.03
+        align_weight_decay: 0
+        arch_specifier: no-align+fused-gelu-mlp
+        enable_gradient_checkpointing: true
+        enable_mixed_precision_training: true
+        finetune_epochs: 2
+        finetune_global_batch_size: 64
+        finetune_learning_rate: 2e-05
+        finetune_lr_scheduler_type: linear-warmup+cosine-decay
+        finetune_max_grad_norm: 1
+        finetune_max_steps: null
+        finetune_per_device_batch_size: 4
+        finetune_save_every_n_steps: 10000
+        finetune_train_strategy: fsdp-full-shard
+        finetune_warmup_ratio: 0.03
+        finetune_weight_decay: 0.1
+        image_resize_strategy: resize-naive
+        llm_backbone_id: qwen25-0_5b-extra
+        llm_max_length: 32768
+        model_id: prism-qwen25-extra-dinosiglip-224px+0_5b
+        reduce_in_full_precision: false
+        type: prism-qwen25-extra-dinosiglip-224px+0_5b
+        vision_backbone_id: dinosiglip-vit-so-224px
+pretrained_checkpoint:
+    value: null
+run_id:
+    value: prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7
+run_root_dir:
+    value: runs
+seed:
+    value: 7
+stage:
+    value: finetune
+trackers:
+    value:
+        - jsonl
+        - wandb
+wandb_entity:
+    value: null
+wandb_project:
+    value: prismatic

wandb/latest-run/files/output.log

@@ -0,0 +1,4 @@
+11/05 [19:31:03] INFO     | >> [*] Starting Training Loop        pretrain.py:227
+                                                                                                                          
+11/05 [23:38:31] INFO     | >> [*] Done with Training =>>        pretrain.py:231
+                          Finalizing Metrics

wandb/latest-run/files/wandb-metadata.json

@@ -0,0 +1,134 @@
+{
+  "os":  "Linux-5.15.0-116-generic-x86_64-with-glibc2.35",
+  "python":  "3.10.15",
+  "startedAt":  "2024-11-06T03:31:02.513650Z",
+  "args":  [
+    "--model.type",
+    "prism-qwen25-extra-dinosiglip-224px+0_5b",
+    "--model.finetune_global_batch_size",
+    "64",
+    "--model.finetune_per_device_batch_size",
+    "4"
+  ],
+  "program":  "/hai/scratch/belkhale/openvla-mini/scripts/pretrain.py",
+  "codePath":  "scripts/pretrain.py",
+  "git":  {
+    "remote":  "git@github.com:Stanford-ILIAD/openvla-mini.git",
+    "commit":  "05073927b096dab7d326a3e39db9262f08d3a8ae"
+  },
+  "email":  "belkhale@stanford.edu",
+  "root":  "runs/prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7",
+  "host":  "haic-hgx-2.stanford.edu",
+  "username":  "belkhale",
+  "executable":  "/hai/scratch/belkhale/miniforge3/envs/vla/bin/python3.10",
+  "codePathLocal":  "scripts/pretrain.py",
+  "cpu_count":  112,
+  "cpu_count_logical":  224,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "942725181440",
+      "used":  "50880847872"
+    }
+  },
+  "memory":  {
+    "total":  "2164104577024"
+  },
+  "cpu":  {
+    "count":  112,
+    "countLogical":  224
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    }
+  ],
+  "slurm":  {
+    "cluster_name":  "haic",
+    "conf":  "/usr/local/etc/slurm.conf",
+    "cpus_on_node":  "64",
+    "cpus_per_task":  "64",
+    "gpus_on_node":  "8",
+    "gtids":  "0",
+    "job_account":  "models",
+    "job_cpus_per_node":  "64",
+    "job_end_time":  "1731122999",
+    "job_gid":  "37",
+    "job_gpus":  "0,1,2,3,4,5,6,7",
+    "job_id":  "11026",
+    "job_name":  "pretrain",
+    "job_nodelist":  "haic-hgx-2",
+    "job_num_nodes":  "1",
+    "job_partition":  "hai",
+    "job_qos":  "models",
+    "job_start_time":  "1730863799",
+    "job_uid":  "377095",
+    "job_user":  "belkhale",
+    "jobid":  "11026",
+    "localid":  "0",
+    "mem_per_node":  "102400",
+    "nnodes":  "1",
+    "nodeid":  "0",
+    "nodelist":  "haic-hgx-2",
+    "nprocs":  "1",
+    "ntasks":  "1",
+    "ntasks_per_node":  "1",
+    "prio_process":  "0",
+    "procid":  "0",
+    "script_context":  "prolog_task",
+    "submit_dir":  "/hai/scratch/belkhale/openvla-mini",
+    "submit_host":  "haic.stanford.edu",
+    "task_pid":  "2187908",
+    "tasks_per_node":  "1",
+    "topology_addr":  "haic-hgx-2",
+    "topology_addr_pattern":  "node",
+    "tres_per_task":  "cpu=64"
+  },
+  "cudaVersion":  "12.4"
+}

wandb/latest-run/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":14849},"_runtime":14849.432571063,"_step":20792,"Finetune/Loss":0.734754204750061,"Finetune/Loss (Raw)":0.7624474763870239,"Finetune/Learning Rate":0,"Finetune/Step Time":0.7374007441103458,"_timestamp":1.7308787040734835e+09,"Finetune/Step":20792}

wandb/latest-run/logs/debug-core.log

@@ -0,0 +1,16 @@
+{"time":"2024-11-05T19:31:02.167132681-08:00","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/belkhale/tmpowkszwq0/port-2188020.txt","pid":2188020,"debug":false,"disable-analytics":false}
+{"time":"2024-11-05T19:31:02.167154904-08:00","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-11-05T19:31:02.168180089-08:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2188020}
+{"time":"2024-11-05T19:31:02.168180088-08:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":35793,"Zone":""}}
+{"time":"2024-11-05T19:31:02.357031058-08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:51484"}
+{"time":"2024-11-05T19:31:02.518306545-08:00","level":"INFO","msg":"handleInformInit: received","streamId":"jcj67gg8","id":"127.0.0.1:51484"}
+{"time":"2024-11-05T19:31:02.738838516-08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"jcj67gg8","id":"127.0.0.1:51484"}
+{"time":"2024-11-05T23:38:34.554528568-08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"jcj67gg8","id":"127.0.0.1:51484"}
+{"time":"2024-11-05T23:38:34.558563756-08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"jcj67gg8","id":"127.0.0.1:51484"}
+{"time":"2024-11-05T23:42:06.504927152-08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:51484"}
+{"time":"2024-11-05T23:42:06.505861575-08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:51484"}
+{"time":"2024-11-05T23:42:06.505880856-08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2024-11-05T23:42:06.50594903-08:00","level":"INFO","msg":"connection: Close: initiating connection closure","id":"127.0.0.1:51484"}
+{"time":"2024-11-05T23:42:06.506095025-08:00","level":"INFO","msg":"connection: Close: connection successfully closed","id":"127.0.0.1:51484"}
+{"time":"2024-11-05T23:42:06.506152479-08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:51484"}
+{"time":"2024-11-05T23:42:06.506171224-08:00","level":"INFO","msg":"server is closed"}

wandb/latest-run/logs/debug-internal.log

@@ -0,0 +1,17 @@
+{"time":"2024-11-05T19:31:02.519998294-08:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-05T19:31:02.520021589-08:00","level":"INFO","msg":"created symlink","path":"runs/prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7/wandb/run-20241105_193102-jcj67gg8/logs/debug-core.log"}
+{"time":"2024-11-05T19:31:02.738801523-08:00","level":"INFO","msg":"created new stream","id":"jcj67gg8"}
+{"time":"2024-11-05T19:31:02.738835256-08:00","level":"INFO","msg":"stream: started","id":"jcj67gg8"}
+{"time":"2024-11-05T19:31:02.738905513-08:00","level":"INFO","msg":"sender: started","stream_id":"jcj67gg8"}
+{"time":"2024-11-05T19:31:02.738892436-08:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"jcj67gg8"}}
+{"time":"2024-11-05T19:31:02.738902832-08:00","level":"INFO","msg":"handler: started","stream_id":{"value":"jcj67gg8"}}
+{"time":"2024-11-05T19:31:03.056521545-08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-05T23:38:31.946246118-08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-05T23:38:32.023973784-08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-05T23:38:32.922341453-08:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"saving job artifact","runtime_seconds":0.775009129}],"total_operations":1}}
+{"time":"2024-11-05T23:38:33.390878425-08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-05T23:38:34.555591357-08:00","level":"INFO","msg":"stream: closing","id":"jcj67gg8"}
+{"time":"2024-11-05T23:38:34.555667186-08:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"jcj67gg8"}}
+{"time":"2024-11-05T23:38:34.555706801-08:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"jcj67gg8"}}
+{"time":"2024-11-05T23:38:34.555872-08:00","level":"INFO","msg":"sender: closed","stream_id":"jcj67gg8"}
+{"time":"2024-11-05T23:38:34.558526671-08:00","level":"INFO","msg":"stream: closed","id":"jcj67gg8"}

wandb/latest-run/logs/debug.log

@@ -0,0 +1,32 @@
+2024-11-05 19:31:02,511 INFO    MainThread:2188020 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-05 19:31:02,511 INFO    MainThread:2188020 [wandb_setup.py:_flush():79] Configure stats pid to 2188020
+2024-11-05 19:31:02,511 INFO    MainThread:2188020 [wandb_setup.py:_flush():79] Loading settings from /hai/scratch/belkhale/.config/wandb/settings
+2024-11-05 19:31:02,511 INFO    MainThread:2188020 [wandb_setup.py:_flush():79] Loading settings from /hai/scratch/belkhale/openvla-mini/wandb/settings
+2024-11-05 19:31:02,511 INFO    MainThread:2188020 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'_service_wait': '300'}
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/hai/scratch/belkhale/openvla-mini/scripts/pretrain.py', 'program': '/hai/scratch/belkhale/openvla-mini/scripts/pretrain.py'}
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_init.py:_log_setup():534] Logging user logs to runs/prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7/wandb/run-20241105_193102-jcj67gg8/logs/debug.log
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_init.py:_log_setup():535] Logging internal logs to runs/prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7/wandb/run-20241105_193102-jcj67gg8/logs/debug-internal.log
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_init.py:init():621] calling init triggers
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {'model': {'type': 'prism-qwen25-extra-dinosiglip-224px+0_5b', 'model_id': 'prism-qwen25-extra-dinosiglip-224px+0_5b', 'arch_specifier': 'no-align+fused-gelu-mlp', 'vision_backbone_id': 'dinosiglip-vit-so-224px', 'llm_backbone_id': 'qwen25-0_5b-extra', 'image_resize_strategy': 'resize-naive', 'llm_max_length': 32768, 'align_epochs': 1, 'align_max_steps': None, 'align_save_every_n_steps': 10000, 'align_global_batch_size': 96, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 2, 'finetune_max_steps': None, 'finetune_save_every_n_steps': 10000, 'finetune_global_batch_size': 64, 'finetune_per_device_batch_size': 4, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'llava-v15', 'dataset_id': 'llava-v15', 'align_stage_components': ['download/llava-laion-cc-sbu-558k/chat.json', 'download/llava-laion-cc-sbu-558k'], 'finetune_stage_components': ['download/llava-v1.5-instruct/llava_v1_5_mix665k.json', 'download/llava-v1.5-instruct'], 'dataset_root_dir': '/hai/scratch/belkhale/datasets/prismatic-vlms'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'hf_token': '.hf_token', 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'prismatic', 'wandb_entity': None}
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_init.py:init():671] starting backend
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_init.py:init():675] sending inform_init request
+2024-11-05 19:31:02,513 INFO    MainThread:2188020 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-05 19:31:02,513 INFO    MainThread:2188020 [wandb_init.py:init():688] backend started and connected
+2024-11-05 19:31:02,515 INFO    MainThread:2188020 [wandb_init.py:init():783] updated telemetry
+2024-11-05 19:31:02,573 INFO    MainThread:2188020 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-05 19:31:03,050 INFO    MainThread:2188020 [wandb_init.py:init():867] starting run threads in backend
+2024-11-05 19:31:03,226 INFO    MainThread:2188020 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-05 19:31:03,227 INFO    MainThread:2188020 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-05 19:31:03,227 INFO    MainThread:2188020 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-05 19:31:03,227 INFO    MainThread:2188020 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-05 19:31:03,230 INFO    MainThread:2188020 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-05 23:38:31,920 INFO    MainThread:2188020 [wandb_run.py:_finish():2158] finishing run belkhale/prismatic/jcj67gg8
+2024-11-05 23:38:31,920 INFO    MainThread:2188020 [wandb_run.py:_atexit_cleanup():2426] got exitcode: 0
+2024-11-05 23:38:31,921 INFO    MainThread:2188020 [wandb_run.py:_restore():2408] restore
+2024-11-05 23:38:31,921 INFO    MainThread:2188020 [wandb_run.py:_restore():2414] restore done
+2024-11-05 23:38:34,516 INFO    MainThread:2188020 [wandb_run.py:_footer_history_summary_info():3975] rendering history
+2024-11-05 23:38:34,517 INFO    MainThread:2188020 [wandb_run.py:_footer_history_summary_info():4007] rendering summary
+2024-11-05 23:38:34,534 INFO    MainThread:2188020 [wandb_run.py:_footer_sync_info():3934] logging synced files

wandb/latest-run/run-jcj67gg8.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1e028b995d945fb180c85455d7219269515a28888eda671f635380e4dac0d23
+size 37709802

wandb/run-20241105_192502-8vxhoj6d/files/config.yaml

@@ -0,0 +1,103 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.10.15
+        t:
+            "1":
+                - 1
+                - 2
+                - 3
+                - 11
+                - 41
+                - 49
+                - 55
+                - 63
+                - 71
+            "2":
+                - 1
+                - 2
+                - 3
+                - 11
+                - 41
+                - 49
+                - 55
+                - 63
+                - 71
+            "3":
+                - 13
+                - 16
+                - 23
+                - 55
+            "4": 3.10.15
+            "5": 0.18.5
+            "6": 4.40.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+dataset:
+    value:
+        align_stage_components:
+            - download/llava-laion-cc-sbu-558k/chat.json
+            - download/llava-laion-cc-sbu-558k
+        dataset_id: llava-v15
+        dataset_root_dir: /hai/scratch/belkhale/datasets/prismatic-vlms
+        finetune_stage_components:
+            - download/llava-v1.5-instruct/llava_v1_5_mix665k.json
+            - download/llava-v1.5-instruct
+        type: llava-v15
+hf_token:
+    value: .hf_token
+model:
+    value:
+        align_epochs: 1
+        align_global_batch_size: 96
+        align_learning_rate: 0.001
+        align_lr_scheduler_type: linear-warmup+cosine-decay
+        align_max_grad_norm: 1
+        align_max_steps: null
+        align_per_device_batch_size: 16
+        align_save_every_n_steps: 10000
+        align_train_strategy: fsdp-shard-grad-op
+        align_warmup_ratio: 0.03
+        align_weight_decay: 0
+        arch_specifier: no-align+fused-gelu-mlp
+        enable_gradient_checkpointing: true
+        enable_mixed_precision_training: true
+        finetune_epochs: 2
+        finetune_global_batch_size: 128
+        finetune_learning_rate: 2e-05
+        finetune_lr_scheduler_type: linear-warmup+cosine-decay
+        finetune_max_grad_norm: 1
+        finetune_max_steps: null
+        finetune_per_device_batch_size: 16
+        finetune_save_every_n_steps: 10000
+        finetune_train_strategy: fsdp-full-shard
+        finetune_warmup_ratio: 0.03
+        finetune_weight_decay: 0.1
+        image_resize_strategy: resize-naive
+        llm_backbone_id: qwen25-0_5b-extra
+        llm_max_length: 32768
+        model_id: prism-qwen25-extra-dinosiglip-224px+0_5b
+        reduce_in_full_precision: false
+        type: prism-qwen25-extra-dinosiglip-224px+0_5b
+        vision_backbone_id: dinosiglip-vit-so-224px
+pretrained_checkpoint:
+    value: null
+run_id:
+    value: prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7
+run_root_dir:
+    value: runs
+seed:
+    value: 7
+stage:
+    value: finetune
+trackers:
+    value:
+        - jsonl
+        - wandb
+wandb_entity:
+    value: null
+wandb_project:
+    value: prismatic

wandb/run-20241105_192502-8vxhoj6d/files/output.log

@@ -0,0 +1,35 @@
+11/05 [19:25:05] INFO     | >> [*] Starting Training Loop        pretrain.py:227
+Traceback (most recent call last):                                                    
+  File "/hai/scratch/belkhale/openvla-mini/scripts/pretrain.py", line 241, in <module>
+    pretrain()
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/draccus/argparsing.py", line 203, in wrapper_inner
+    response = fn(cfg, *args, **kwargs)
+  File "/hai/scratch/belkhale/openvla-mini/scripts/pretrain.py", line 228, in pretrain
+    train_strategy.run_training(train_dataset, collator, metrics, stage=cfg.stage, seed=cfg.seed)
+  File "/hai/scratch/belkhale/openvla-mini/prismatic/training/strategies/base_strategy.py", line 190, in run_training
+    output: CausalLMOutputWithPast = self.vlm(
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py", line 849, in forward
+    output = self._fsdp_wrapped_module(*args, **kwargs)
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/hai/scratch/belkhale/openvla-mini/prismatic/models/vlms/prismatic.py", line 470, in forward
+    return self.llm_backbone(
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/hai/scratch/belkhale/openvla-mini/prismatic/models/backbones/llm/base_llm.py", line 221, in forward
+    output: CausalLMOutputWithPast = self.llm(
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py", line 1183, in forward
+    logits = logits.float()
+torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 49.14 GiB. GPU 0 has a total capacity of 79.10 GiB of which 42.31 GiB is free. Including non-PyTorch memory, this process has 36.77 GiB memory in use. Of the allocated memory 29.84 GiB is allocated by PyTorch, and 1.02 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)

wandb/run-20241105_192502-8vxhoj6d/files/wandb-metadata.json

@@ -0,0 +1,134 @@
+{
+  "os":  "Linux-5.15.0-116-generic-x86_64-with-glibc2.35",
+  "python":  "3.10.15",
+  "startedAt":  "2024-11-06T03:25:02.892307Z",
+  "args":  [
+    "--model.type",
+    "prism-qwen25-extra-dinosiglip-224px+0_5b",
+    "--model.finetune_global_batch_size",
+    "128",
+    "--model.finetune_per_device_batch_size",
+    "16"
+  ],
+  "program":  "/hai/scratch/belkhale/openvla-mini/scripts/pretrain.py",
+  "codePath":  "scripts/pretrain.py",
+  "git":  {
+    "remote":  "git@github.com:Stanford-ILIAD/openvla-mini.git",
+    "commit":  "05073927b096dab7d326a3e39db9262f08d3a8ae"
+  },
+  "email":  "belkhale@stanford.edu",
+  "root":  "runs/prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7",
+  "host":  "haic-hgx-2.stanford.edu",
+  "username":  "belkhale",
+  "executable":  "/hai/scratch/belkhale/miniforge3/envs/vla/bin/python3.10",
+  "codePathLocal":  "scripts/pretrain.py",
+  "cpu_count":  112,
+  "cpu_count_logical":  224,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "942725181440",
+      "used":  "50880245760"
+    }
+  },
+  "memory":  {
+    "total":  "2164104577024"
+  },
+  "cpu":  {
+    "count":  112,
+    "countLogical":  224
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    }
+  ],
+  "slurm":  {
+    "cluster_name":  "haic",
+    "conf":  "/usr/local/etc/slurm.conf",
+    "cpus_on_node":  "64",
+    "cpus_per_task":  "64",
+    "gpus_on_node":  "8",
+    "gtids":  "0",
+    "job_account":  "models",
+    "job_cpus_per_node":  "64",
+    "job_end_time":  "1731122631",
+    "job_gid":  "37",
+    "job_gpus":  "0,1,2,3,4,5,6,7",
+    "job_id":  "11023",
+    "job_name":  "pretrain",
+    "job_nodelist":  "haic-hgx-2",
+    "job_num_nodes":  "1",
+    "job_partition":  "hai",
+    "job_qos":  "models",
+    "job_start_time":  "1730863431",
+    "job_uid":  "377095",
+    "job_user":  "belkhale",
+    "jobid":  "11023",
+    "localid":  "0",
+    "mem_per_node":  "102400",
+    "nnodes":  "1",
+    "nodeid":  "0",
+    "nodelist":  "haic-hgx-2",
+    "nprocs":  "1",
+    "ntasks":  "1",
+    "ntasks_per_node":  "1",
+    "prio_process":  "0",
+    "procid":  "0",
+    "script_context":  "prolog_task",
+    "submit_dir":  "/hai/scratch/belkhale/openvla-mini",
+    "submit_host":  "haic.stanford.edu",
+    "task_pid":  "2182609",
+    "tasks_per_node":  "1",
+    "topology_addr":  "haic-hgx-2",
+    "topology_addr_pattern":  "node",
+    "tres_per_task":  "cpu=64"
+  },
+  "cudaVersion":  "12.4"
+}

wandb/run-20241105_192502-8vxhoj6d/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":22}}

wandb/run-20241105_192502-8vxhoj6d/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2024-11-05T19:25:02.690703685-08:00","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/belkhale/tmp97kp6p__/port-2182724.txt","pid":2182724,"debug":false,"disable-analytics":false}
+{"time":"2024-11-05T19:25:02.690734566-08:00","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-11-05T19:25:02.692100649-08:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2182724}
+{"time":"2024-11-05T19:25:02.69208955-08:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":41525,"Zone":""}}
+{"time":"2024-11-05T19:25:02.744814731-08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:59560"}
+{"time":"2024-11-05T19:25:02.895430941-08:00","level":"INFO","msg":"handleInformInit: received","streamId":"8vxhoj6d","id":"127.0.0.1:59560"}
+{"time":"2024-11-05T19:25:03.194511518-08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"8vxhoj6d","id":"127.0.0.1:59560"}
+{"time":"2024-11-05T19:25:25.782400709-08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:59560"}
+{"time":"2024-11-05T19:25:25.782779877-08:00","level":"INFO","msg":"connection: Close: initiating connection closure","id":"127.0.0.1:59560"}
+{"time":"2024-11-05T19:25:25.782799272-08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2024-11-05T19:25:25.783947145-08:00","level":"INFO","msg":"connection: Close: connection successfully closed","id":"127.0.0.1:59560"}
+{"time":"2024-11-05T19:25:27.304522773-08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:59560"}
+{"time":"2024-11-05T19:25:27.304560668-08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:59560"}
+{"time":"2024-11-05T19:25:27.304578483-08:00","level":"INFO","msg":"server is closed"}

wandb/run-20241105_192502-8vxhoj6d/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2024-11-05T19:25:02.896392755-08:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-05T19:25:02.896416546-08:00","level":"INFO","msg":"created symlink","path":"runs/prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7/wandb/run-20241105_192502-8vxhoj6d/logs/debug-core.log"}
+{"time":"2024-11-05T19:25:03.194464738-08:00","level":"INFO","msg":"created new stream","id":"8vxhoj6d"}
+{"time":"2024-11-05T19:25:03.194508127-08:00","level":"INFO","msg":"stream: started","id":"8vxhoj6d"}
+{"time":"2024-11-05T19:25:03.194578475-08:00","level":"INFO","msg":"sender: started","stream_id":"8vxhoj6d"}
+{"time":"2024-11-05T19:25:03.194561979-08:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"8vxhoj6d"}}
+{"time":"2024-11-05T19:25:03.194595326-08:00","level":"INFO","msg":"handler: started","stream_id":{"value":"8vxhoj6d"}}
+{"time":"2024-11-05T19:25:03.382215025-08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-05T19:25:25.782775323-08:00","level":"INFO","msg":"stream: closing","id":"8vxhoj6d"}
+{"time":"2024-11-05T19:25:25.784095739-08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-05T19:25:25.785772774-08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-05T19:25:27.185173962-08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-05T19:25:27.303068441-08:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"8vxhoj6d"}}
+{"time":"2024-11-05T19:25:27.303126471-08:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"8vxhoj6d"}}
+{"time":"2024-11-05T19:25:27.303147247-08:00","level":"INFO","msg":"sender: closed","stream_id":"8vxhoj6d"}
+{"time":"2024-11-05T19:25:27.304415448-08:00","level":"INFO","msg":"stream: closed","id":"8vxhoj6d"}

wandb/run-20241105_192502-8vxhoj6d/logs/debug.log

@@ -0,0 +1,26 @@
+2024-11-05 19:25:02,890 INFO    MainThread:2182724 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-05 19:25:02,890 INFO    MainThread:2182724 [wandb_setup.py:_flush():79] Configure stats pid to 2182724
+2024-11-05 19:25:02,890 INFO    MainThread:2182724 [wandb_setup.py:_flush():79] Loading settings from /hai/scratch/belkhale/.config/wandb/settings
+2024-11-05 19:25:02,890 INFO    MainThread:2182724 [wandb_setup.py:_flush():79] Loading settings from /hai/scratch/belkhale/openvla-mini/wandb/settings
+2024-11-05 19:25:02,890 INFO    MainThread:2182724 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'_service_wait': '300'}
+2024-11-05 19:25:02,890 INFO    MainThread:2182724 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-05 19:25:02,890 INFO    MainThread:2182724 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/hai/scratch/belkhale/openvla-mini/scripts/pretrain.py', 'program': '/hai/scratch/belkhale/openvla-mini/scripts/pretrain.py'}
+2024-11-05 19:25:02,890 INFO    MainThread:2182724 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-05 19:25:02,890 INFO    MainThread:2182724 [wandb_init.py:_log_setup():534] Logging user logs to runs/prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7/wandb/run-20241105_192502-8vxhoj6d/logs/debug.log
+2024-11-05 19:25:02,890 INFO    MainThread:2182724 [wandb_init.py:_log_setup():535] Logging internal logs to runs/prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7/wandb/run-20241105_192502-8vxhoj6d/logs/debug-internal.log
+2024-11-05 19:25:02,890 INFO    MainThread:2182724 [wandb_init.py:init():621] calling init triggers
+2024-11-05 19:25:02,890 INFO    MainThread:2182724 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {'model': {'type': 'prism-qwen25-extra-dinosiglip-224px+0_5b', 'model_id': 'prism-qwen25-extra-dinosiglip-224px+0_5b', 'arch_specifier': 'no-align+fused-gelu-mlp', 'vision_backbone_id': 'dinosiglip-vit-so-224px', 'llm_backbone_id': 'qwen25-0_5b-extra', 'image_resize_strategy': 'resize-naive', 'llm_max_length': 32768, 'align_epochs': 1, 'align_max_steps': None, 'align_save_every_n_steps': 10000, 'align_global_batch_size': 96, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 2, 'finetune_max_steps': None, 'finetune_save_every_n_steps': 10000, 'finetune_global_batch_size': 128, 'finetune_per_device_batch_size': 16, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'llava-v15', 'dataset_id': 'llava-v15', 'align_stage_components': ['download/llava-laion-cc-sbu-558k/chat.json', 'download/llava-laion-cc-sbu-558k'], 'finetune_stage_components': ['download/llava-v1.5-instruct/llava_v1_5_mix665k.json', 'download/llava-v1.5-instruct'], 'dataset_root_dir': '/hai/scratch/belkhale/datasets/prismatic-vlms'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'hf_token': '.hf_token', 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'prismatic', 'wandb_entity': None}
+2024-11-05 19:25:02,890 INFO    MainThread:2182724 [wandb_init.py:init():671] starting backend
+2024-11-05 19:25:02,890 INFO    MainThread:2182724 [wandb_init.py:init():675] sending inform_init request
+2024-11-05 19:25:02,891 INFO    MainThread:2182724 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-05 19:25:02,892 INFO    MainThread:2182724 [wandb_init.py:init():688] backend started and connected
+2024-11-05 19:25:02,893 INFO    MainThread:2182724 [wandb_init.py:init():783] updated telemetry
+2024-11-05 19:25:03,020 INFO    MainThread:2182724 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-05 19:25:03,377 INFO    MainThread:2182724 [wandb_init.py:init():867] starting run threads in backend
+2024-11-05 19:25:05,102 INFO    MainThread:2182724 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-05 19:25:05,102 INFO    MainThread:2182724 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-05 19:25:05,103 INFO    MainThread:2182724 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-05 19:25:05,103 INFO    MainThread:2182724 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-05 19:25:05,128 INFO    MainThread:2182724 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-05 19:25:25,783 WARNING MsgRouterThr:2182724 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241105_192659-mqdqjqly/files/config.yaml

@@ -0,0 +1,103 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.10.15
+        t:
+            "1":
+                - 1
+                - 2
+                - 3
+                - 11
+                - 41
+                - 49
+                - 55
+                - 63
+                - 71
+            "2":
+                - 1
+                - 2
+                - 3
+                - 11
+                - 41
+                - 49
+                - 55
+                - 63
+                - 71
+            "3":
+                - 13
+                - 16
+                - 23
+                - 55
+            "4": 3.10.15
+            "5": 0.18.5
+            "6": 4.40.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+dataset:
+    value:
+        align_stage_components:
+            - download/llava-laion-cc-sbu-558k/chat.json
+            - download/llava-laion-cc-sbu-558k
+        dataset_id: llava-v15
+        dataset_root_dir: /hai/scratch/belkhale/datasets/prismatic-vlms
+        finetune_stage_components:
+            - download/llava-v1.5-instruct/llava_v1_5_mix665k.json
+            - download/llava-v1.5-instruct
+        type: llava-v15
+hf_token:
+    value: .hf_token
+model:
+    value:
+        align_epochs: 1
+        align_global_batch_size: 96
+        align_learning_rate: 0.001
+        align_lr_scheduler_type: linear-warmup+cosine-decay
+        align_max_grad_norm: 1
+        align_max_steps: null
+        align_per_device_batch_size: 16
+        align_save_every_n_steps: 10000
+        align_train_strategy: fsdp-shard-grad-op
+        align_warmup_ratio: 0.03
+        align_weight_decay: 0
+        arch_specifier: no-align+fused-gelu-mlp
+        enable_gradient_checkpointing: true
+        enable_mixed_precision_training: true
+        finetune_epochs: 2
+        finetune_global_batch_size: 64
+        finetune_learning_rate: 2e-05
+        finetune_lr_scheduler_type: linear-warmup+cosine-decay
+        finetune_max_grad_norm: 1
+        finetune_max_steps: null
+        finetune_per_device_batch_size: 8
+        finetune_save_every_n_steps: 10000
+        finetune_train_strategy: fsdp-full-shard
+        finetune_warmup_ratio: 0.03
+        finetune_weight_decay: 0.1
+        image_resize_strategy: resize-naive
+        llm_backbone_id: qwen25-0_5b-extra
+        llm_max_length: 32768
+        model_id: prism-qwen25-extra-dinosiglip-224px+0_5b
+        reduce_in_full_precision: false
+        type: prism-qwen25-extra-dinosiglip-224px+0_5b
+        vision_backbone_id: dinosiglip-vit-so-224px
+pretrained_checkpoint:
+    value: null
+run_id:
+    value: prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7
+run_root_dir:
+    value: runs
+seed:
+    value: 7
+stage:
+    value: finetune
+trackers:
+    value:
+        - jsonl
+        - wandb
+wandb_entity:
+    value: null
+wandb_project:
+    value: prismatic

wandb/run-20241105_192659-mqdqjqly/files/output.log

@@ -0,0 +1,43 @@
+11/05 [19:26:59] INFO     | >> [*] Starting Training Loop        pretrain.py:227
+Traceback (most recent call last):                                                    
+  File "/hai/scratch/belkhale/openvla-mini/scripts/pretrain.py", line 241, in <module>
+    pretrain()
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/draccus/argparsing.py", line 203, in wrapper_inner
+    response = fn(cfg, *args, **kwargs)
+  File "/hai/scratch/belkhale/openvla-mini/scripts/pretrain.py", line 228, in pretrain
+    train_strategy.run_training(train_dataset, collator, metrics, stage=cfg.stage, seed=cfg.seed)
+  File "/hai/scratch/belkhale/openvla-mini/prismatic/training/strategies/base_strategy.py", line 190, in run_training
+    output: CausalLMOutputWithPast = self.vlm(
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py", line 849, in forward
+    output = self._fsdp_wrapped_module(*args, **kwargs)
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/hai/scratch/belkhale/openvla-mini/prismatic/models/vlms/prismatic.py", line 470, in forward
+    return self.llm_backbone(
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/hai/scratch/belkhale/openvla-mini/prismatic/models/backbones/llm/base_llm.py", line 221, in forward
+    output: CausalLMOutputWithPast = self.llm(
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/transformers/models/qwen2/modeling_qwen2.py", line 1196, in forward
+    loss = loss_fct(shift_logits, shift_labels)
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1520, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/torch/nn/modules/loss.py", line 1179, in forward
+    return F.cross_entropy(input, target, weight=self.weight,
+  File "/hai/scratch/belkhale/miniforge3/envs/vla/lib/python3.10/site-packages/torch/nn/functional.py", line 3059, in cross_entropy
+    return torch._C._nn.cross_entropy_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index, label_smoothing)
+torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 24.57 GiB. GPU 0 has a total capacity of 79.10 GiB of which 20.18 GiB is free. Including non-PyTorch memory, this process has 58.91 GiB memory in use. Of the allocated memory 52.22 GiB is allocated by PyTorch, and 798.66 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)

wandb/run-20241105_192659-mqdqjqly/files/wandb-metadata.json

@@ -0,0 +1,134 @@
+{
+  "os":  "Linux-5.15.0-116-generic-x86_64-with-glibc2.35",
+  "python":  "3.10.15",
+  "startedAt":  "2024-11-06T03:26:59.309206Z",
+  "args":  [
+    "--model.type",
+    "prism-qwen25-extra-dinosiglip-224px+0_5b",
+    "--model.finetune_global_batch_size",
+    "64",
+    "--model.finetune_per_device_batch_size",
+    "8"
+  ],
+  "program":  "/hai/scratch/belkhale/openvla-mini/scripts/pretrain.py",
+  "codePath":  "scripts/pretrain.py",
+  "git":  {
+    "remote":  "git@github.com:Stanford-ILIAD/openvla-mini.git",
+    "commit":  "05073927b096dab7d326a3e39db9262f08d3a8ae"
+  },
+  "email":  "belkhale@stanford.edu",
+  "root":  "runs/prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7",
+  "host":  "haic-hgx-2.stanford.edu",
+  "username":  "belkhale",
+  "executable":  "/hai/scratch/belkhale/miniforge3/envs/vla/bin/python3.10",
+  "codePathLocal":  "scripts/pretrain.py",
+  "cpu_count":  112,
+  "cpu_count_logical":  224,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "942725181440",
+      "used":  "50880540672"
+    }
+  },
+  "memory":  {
+    "total":  "2164104577024"
+  },
+  "cpu":  {
+    "count":  112,
+    "countLogical":  224
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    }
+  ],
+  "slurm":  {
+    "cluster_name":  "haic",
+    "conf":  "/usr/local/etc/slurm.conf",
+    "cpus_on_node":  "64",
+    "cpus_per_task":  "64",
+    "gpus_on_node":  "8",
+    "gtids":  "0",
+    "job_account":  "models",
+    "job_cpus_per_node":  "64",
+    "job_end_time":  "1731122748",
+    "job_gid":  "37",
+    "job_gpus":  "0,1,2,3,4,5,6,7",
+    "job_id":  "11024",
+    "job_name":  "pretrain",
+    "job_nodelist":  "haic-hgx-2",
+    "job_num_nodes":  "1",
+    "job_partition":  "hai",
+    "job_qos":  "models",
+    "job_start_time":  "1730863548",
+    "job_uid":  "377095",
+    "job_user":  "belkhale",
+    "jobid":  "11024",
+    "localid":  "0",
+    "mem_per_node":  "102400",
+    "nnodes":  "1",
+    "nodeid":  "0",
+    "nodelist":  "haic-hgx-2",
+    "nprocs":  "1",
+    "ntasks":  "1",
+    "ntasks_per_node":  "1",
+    "prio_process":  "0",
+    "procid":  "0",
+    "script_context":  "prolog_task",
+    "submit_dir":  "/hai/scratch/belkhale/openvla-mini",
+    "submit_host":  "haic.stanford.edu",
+    "task_pid":  "2184784",
+    "tasks_per_node":  "1",
+    "topology_addr":  "haic-hgx-2",
+    "topology_addr_pattern":  "node",
+    "tres_per_task":  "cpu=64"
+  },
+  "cudaVersion":  "12.4"
+}

wandb/run-20241105_192659-mqdqjqly/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":18}}

wandb/run-20241105_192659-mqdqjqly/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2024-11-05T19:26:58.953439705-08:00","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/belkhale/tmp1fqrk6pa/port-2184904.txt","pid":2184904,"debug":false,"disable-analytics":false}
+{"time":"2024-11-05T19:26:58.953464408-08:00","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-11-05T19:26:58.954979044-08:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":41281,"Zone":""}}
+{"time":"2024-11-05T19:26:58.955028412-08:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2184904}
+{"time":"2024-11-05T19:26:59.144036286-08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:38180"}
+{"time":"2024-11-05T19:26:59.313342183-08:00","level":"INFO","msg":"handleInformInit: received","streamId":"mqdqjqly","id":"127.0.0.1:38180"}
+{"time":"2024-11-05T19:26:59.533375776-08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"mqdqjqly","id":"127.0.0.1:38180"}
+{"time":"2024-11-05T19:27:18.207609388-08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:38180"}
+{"time":"2024-11-05T19:27:18.208156833-08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2024-11-05T19:27:18.20815625-08:00","level":"INFO","msg":"connection: Close: initiating connection closure","id":"127.0.0.1:38180"}
+{"time":"2024-11-05T19:27:18.208292453-08:00","level":"INFO","msg":"connection: Close: connection successfully closed","id":"127.0.0.1:38180"}
+{"time":"2024-11-05T19:27:18.887422598-08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:38180"}
+{"time":"2024-11-05T19:27:18.887444653-08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:38180"}
+{"time":"2024-11-05T19:27:18.887460329-08:00","level":"INFO","msg":"server is closed"}

wandb/run-20241105_192659-mqdqjqly/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2024-11-05T19:26:59.314379425-08:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-05T19:26:59.314391049-08:00","level":"INFO","msg":"created symlink","path":"runs/prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7/wandb/run-20241105_192659-mqdqjqly/logs/debug-core.log"}
+{"time":"2024-11-05T19:26:59.533339524-08:00","level":"INFO","msg":"created new stream","id":"mqdqjqly"}
+{"time":"2024-11-05T19:26:59.533372406-08:00","level":"INFO","msg":"stream: started","id":"mqdqjqly"}
+{"time":"2024-11-05T19:26:59.533420404-08:00","level":"INFO","msg":"sender: started","stream_id":"mqdqjqly"}
+{"time":"2024-11-05T19:26:59.533413693-08:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"mqdqjqly"}}
+{"time":"2024-11-05T19:26:59.533432721-08:00","level":"INFO","msg":"handler: started","stream_id":{"value":"mqdqjqly"}}
+{"time":"2024-11-05T19:26:59.715136674-08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-05T19:27:18.2081844-08:00","level":"INFO","msg":"stream: closing","id":"mqdqjqly"}
+{"time":"2024-11-05T19:27:18.20827275-08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-05T19:27:18.209227648-08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-05T19:27:18.690761255-08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-05T19:27:18.88603778-08:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"mqdqjqly"}}
+{"time":"2024-11-05T19:27:18.886115295-08:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"mqdqjqly"}}
+{"time":"2024-11-05T19:27:18.88614639-08:00","level":"INFO","msg":"sender: closed","stream_id":"mqdqjqly"}
+{"time":"2024-11-05T19:27:18.887360713-08:00","level":"INFO","msg":"stream: closed","id":"mqdqjqly"}

wandb/run-20241105_192659-mqdqjqly/logs/debug.log

@@ -0,0 +1,26 @@
+2024-11-05 19:26:59,307 INFO    MainThread:2184904 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-05 19:26:59,307 INFO    MainThread:2184904 [wandb_setup.py:_flush():79] Configure stats pid to 2184904
+2024-11-05 19:26:59,307 INFO    MainThread:2184904 [wandb_setup.py:_flush():79] Loading settings from /hai/scratch/belkhale/.config/wandb/settings
+2024-11-05 19:26:59,307 INFO    MainThread:2184904 [wandb_setup.py:_flush():79] Loading settings from /hai/scratch/belkhale/openvla-mini/wandb/settings
+2024-11-05 19:26:59,307 INFO    MainThread:2184904 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'_service_wait': '300'}
+2024-11-05 19:26:59,307 INFO    MainThread:2184904 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-05 19:26:59,307 INFO    MainThread:2184904 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/hai/scratch/belkhale/openvla-mini/scripts/pretrain.py', 'program': '/hai/scratch/belkhale/openvla-mini/scripts/pretrain.py'}
+2024-11-05 19:26:59,307 INFO    MainThread:2184904 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-05 19:26:59,307 INFO    MainThread:2184904 [wandb_init.py:_log_setup():534] Logging user logs to runs/prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7/wandb/run-20241105_192659-mqdqjqly/logs/debug.log
+2024-11-05 19:26:59,307 INFO    MainThread:2184904 [wandb_init.py:_log_setup():535] Logging internal logs to runs/prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7/wandb/run-20241105_192659-mqdqjqly/logs/debug-internal.log
+2024-11-05 19:26:59,307 INFO    MainThread:2184904 [wandb_init.py:init():621] calling init triggers
+2024-11-05 19:26:59,307 INFO    MainThread:2184904 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {'model': {'type': 'prism-qwen25-extra-dinosiglip-224px+0_5b', 'model_id': 'prism-qwen25-extra-dinosiglip-224px+0_5b', 'arch_specifier': 'no-align+fused-gelu-mlp', 'vision_backbone_id': 'dinosiglip-vit-so-224px', 'llm_backbone_id': 'qwen25-0_5b-extra', 'image_resize_strategy': 'resize-naive', 'llm_max_length': 32768, 'align_epochs': 1, 'align_max_steps': None, 'align_save_every_n_steps': 10000, 'align_global_batch_size': 96, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 2, 'finetune_max_steps': None, 'finetune_save_every_n_steps': 10000, 'finetune_global_batch_size': 64, 'finetune_per_device_batch_size': 8, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'llava-v15', 'dataset_id': 'llava-v15', 'align_stage_components': ['download/llava-laion-cc-sbu-558k/chat.json', 'download/llava-laion-cc-sbu-558k'], 'finetune_stage_components': ['download/llava-v1.5-instruct/llava_v1_5_mix665k.json', 'download/llava-v1.5-instruct'], 'dataset_root_dir': '/hai/scratch/belkhale/datasets/prismatic-vlms'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'hf_token': '.hf_token', 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'prismatic', 'wandb_entity': None}
+2024-11-05 19:26:59,307 INFO    MainThread:2184904 [wandb_init.py:init():671] starting backend
+2024-11-05 19:26:59,307 INFO    MainThread:2184904 [wandb_init.py:init():675] sending inform_init request
+2024-11-05 19:26:59,308 INFO    MainThread:2184904 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-05 19:26:59,309 INFO    MainThread:2184904 [wandb_init.py:init():688] backend started and connected
+2024-11-05 19:26:59,310 INFO    MainThread:2184904 [wandb_init.py:init():783] updated telemetry
+2024-11-05 19:26:59,372 INFO    MainThread:2184904 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-05 19:26:59,708 INFO    MainThread:2184904 [wandb_init.py:init():867] starting run threads in backend
+2024-11-05 19:26:59,921 INFO    MainThread:2184904 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-05 19:26:59,921 INFO    MainThread:2184904 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-05 19:26:59,921 INFO    MainThread:2184904 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-05 19:26:59,921 INFO    MainThread:2184904 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-05 19:26:59,924 INFO    MainThread:2184904 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-05 19:27:18,208 WARNING MsgRouterThr:2184904 [router.py:message_loop():77] message_loop has been closed

wandb/run-20241105_193102-jcj67gg8/files/config.yaml

@@ -0,0 +1,105 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.10.15
+        t:
+            "1":
+                - 1
+                - 2
+                - 3
+                - 11
+                - 41
+                - 49
+                - 55
+                - 63
+                - 71
+            "2":
+                - 1
+                - 2
+                - 3
+                - 11
+                - 41
+                - 49
+                - 55
+                - 63
+                - 71
+            "3":
+                - 2
+                - 13
+                - 16
+                - 23
+                - 55
+                - 61
+            "4": 3.10.15
+            "5": 0.18.5
+            "6": 4.40.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+dataset:
+    value:
+        align_stage_components:
+            - download/llava-laion-cc-sbu-558k/chat.json
+            - download/llava-laion-cc-sbu-558k
+        dataset_id: llava-v15
+        dataset_root_dir: /hai/scratch/belkhale/datasets/prismatic-vlms
+        finetune_stage_components:
+            - download/llava-v1.5-instruct/llava_v1_5_mix665k.json
+            - download/llava-v1.5-instruct
+        type: llava-v15
+hf_token:
+    value: .hf_token
+model:
+    value:
+        align_epochs: 1
+        align_global_batch_size: 96
+        align_learning_rate: 0.001
+        align_lr_scheduler_type: linear-warmup+cosine-decay
+        align_max_grad_norm: 1
+        align_max_steps: null
+        align_per_device_batch_size: 16
+        align_save_every_n_steps: 10000
+        align_train_strategy: fsdp-shard-grad-op
+        align_warmup_ratio: 0.03
+        align_weight_decay: 0
+        arch_specifier: no-align+fused-gelu-mlp
+        enable_gradient_checkpointing: true
+        enable_mixed_precision_training: true
+        finetune_epochs: 2
+        finetune_global_batch_size: 64
+        finetune_learning_rate: 2e-05
+        finetune_lr_scheduler_type: linear-warmup+cosine-decay
+        finetune_max_grad_norm: 1
+        finetune_max_steps: null
+        finetune_per_device_batch_size: 4
+        finetune_save_every_n_steps: 10000
+        finetune_train_strategy: fsdp-full-shard
+        finetune_warmup_ratio: 0.03
+        finetune_weight_decay: 0.1
+        image_resize_strategy: resize-naive
+        llm_backbone_id: qwen25-0_5b-extra
+        llm_max_length: 32768
+        model_id: prism-qwen25-extra-dinosiglip-224px+0_5b
+        reduce_in_full_precision: false
+        type: prism-qwen25-extra-dinosiglip-224px+0_5b
+        vision_backbone_id: dinosiglip-vit-so-224px
+pretrained_checkpoint:
+    value: null
+run_id:
+    value: prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7
+run_root_dir:
+    value: runs
+seed:
+    value: 7
+stage:
+    value: finetune
+trackers:
+    value:
+        - jsonl
+        - wandb
+wandb_entity:
+    value: null
+wandb_project:
+    value: prismatic

wandb/run-20241105_193102-jcj67gg8/files/output.log

@@ -0,0 +1,4 @@
+11/05 [19:31:03] INFO     | >> [*] Starting Training Loop        pretrain.py:227
+                                                                                                                          
+11/05 [23:38:31] INFO     | >> [*] Done with Training =>>        pretrain.py:231
+                          Finalizing Metrics

wandb/run-20241105_193102-jcj67gg8/files/wandb-metadata.json

@@ -0,0 +1,134 @@
+{
+  "os":  "Linux-5.15.0-116-generic-x86_64-with-glibc2.35",
+  "python":  "3.10.15",
+  "startedAt":  "2024-11-06T03:31:02.513650Z",
+  "args":  [
+    "--model.type",
+    "prism-qwen25-extra-dinosiglip-224px+0_5b",
+    "--model.finetune_global_batch_size",
+    "64",
+    "--model.finetune_per_device_batch_size",
+    "4"
+  ],
+  "program":  "/hai/scratch/belkhale/openvla-mini/scripts/pretrain.py",
+  "codePath":  "scripts/pretrain.py",
+  "git":  {
+    "remote":  "git@github.com:Stanford-ILIAD/openvla-mini.git",
+    "commit":  "05073927b096dab7d326a3e39db9262f08d3a8ae"
+  },
+  "email":  "belkhale@stanford.edu",
+  "root":  "runs/prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7",
+  "host":  "haic-hgx-2.stanford.edu",
+  "username":  "belkhale",
+  "executable":  "/hai/scratch/belkhale/miniforge3/envs/vla/bin/python3.10",
+  "codePathLocal":  "scripts/pretrain.py",
+  "cpu_count":  112,
+  "cpu_count_logical":  224,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "942725181440",
+      "used":  "50880847872"
+    }
+  },
+  "memory":  {
+    "total":  "2164104577024"
+  },
+  "cpu":  {
+    "count":  112,
+    "countLogical":  224
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    }
+  ],
+  "slurm":  {
+    "cluster_name":  "haic",
+    "conf":  "/usr/local/etc/slurm.conf",
+    "cpus_on_node":  "64",
+    "cpus_per_task":  "64",
+    "gpus_on_node":  "8",
+    "gtids":  "0",
+    "job_account":  "models",
+    "job_cpus_per_node":  "64",
+    "job_end_time":  "1731122999",
+    "job_gid":  "37",
+    "job_gpus":  "0,1,2,3,4,5,6,7",
+    "job_id":  "11026",
+    "job_name":  "pretrain",
+    "job_nodelist":  "haic-hgx-2",
+    "job_num_nodes":  "1",
+    "job_partition":  "hai",
+    "job_qos":  "models",
+    "job_start_time":  "1730863799",
+    "job_uid":  "377095",
+    "job_user":  "belkhale",
+    "jobid":  "11026",
+    "localid":  "0",
+    "mem_per_node":  "102400",
+    "nnodes":  "1",
+    "nodeid":  "0",
+    "nodelist":  "haic-hgx-2",
+    "nprocs":  "1",
+    "ntasks":  "1",
+    "ntasks_per_node":  "1",
+    "prio_process":  "0",
+    "procid":  "0",
+    "script_context":  "prolog_task",
+    "submit_dir":  "/hai/scratch/belkhale/openvla-mini",
+    "submit_host":  "haic.stanford.edu",
+    "task_pid":  "2187908",
+    "tasks_per_node":  "1",
+    "topology_addr":  "haic-hgx-2",
+    "topology_addr_pattern":  "node",
+    "tres_per_task":  "cpu=64"
+  },
+  "cudaVersion":  "12.4"
+}

wandb/run-20241105_193102-jcj67gg8/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":14849},"_runtime":14849.432571063,"_step":20792,"Finetune/Loss":0.734754204750061,"Finetune/Loss (Raw)":0.7624474763870239,"Finetune/Learning Rate":0,"Finetune/Step Time":0.7374007441103458,"_timestamp":1.7308787040734835e+09,"Finetune/Step":20792}

wandb/run-20241105_193102-jcj67gg8/logs/debug-core.log

@@ -0,0 +1,16 @@
+{"time":"2024-11-05T19:31:02.167132681-08:00","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/belkhale/tmpowkszwq0/port-2188020.txt","pid":2188020,"debug":false,"disable-analytics":false}
+{"time":"2024-11-05T19:31:02.167154904-08:00","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-11-05T19:31:02.168180089-08:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2188020}
+{"time":"2024-11-05T19:31:02.168180088-08:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":35793,"Zone":""}}
+{"time":"2024-11-05T19:31:02.357031058-08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:51484"}
+{"time":"2024-11-05T19:31:02.518306545-08:00","level":"INFO","msg":"handleInformInit: received","streamId":"jcj67gg8","id":"127.0.0.1:51484"}
+{"time":"2024-11-05T19:31:02.738838516-08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"jcj67gg8","id":"127.0.0.1:51484"}
+{"time":"2024-11-05T23:38:34.554528568-08:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"jcj67gg8","id":"127.0.0.1:51484"}
+{"time":"2024-11-05T23:38:34.558563756-08:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"jcj67gg8","id":"127.0.0.1:51484"}
+{"time":"2024-11-05T23:42:06.504927152-08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:51484"}
+{"time":"2024-11-05T23:42:06.505861575-08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:51484"}
+{"time":"2024-11-05T23:42:06.505880856-08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2024-11-05T23:42:06.50594903-08:00","level":"INFO","msg":"connection: Close: initiating connection closure","id":"127.0.0.1:51484"}
+{"time":"2024-11-05T23:42:06.506095025-08:00","level":"INFO","msg":"connection: Close: connection successfully closed","id":"127.0.0.1:51484"}
+{"time":"2024-11-05T23:42:06.506152479-08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:51484"}
+{"time":"2024-11-05T23:42:06.506171224-08:00","level":"INFO","msg":"server is closed"}

wandb/run-20241105_193102-jcj67gg8/logs/debug-internal.log

@@ -0,0 +1,17 @@
+{"time":"2024-11-05T19:31:02.519998294-08:00","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2024-11-05T19:31:02.520021589-08:00","level":"INFO","msg":"created symlink","path":"runs/prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7/wandb/run-20241105_193102-jcj67gg8/logs/debug-core.log"}
+{"time":"2024-11-05T19:31:02.738801523-08:00","level":"INFO","msg":"created new stream","id":"jcj67gg8"}
+{"time":"2024-11-05T19:31:02.738835256-08:00","level":"INFO","msg":"stream: started","id":"jcj67gg8"}
+{"time":"2024-11-05T19:31:02.738905513-08:00","level":"INFO","msg":"sender: started","stream_id":"jcj67gg8"}
+{"time":"2024-11-05T19:31:02.738892436-08:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"jcj67gg8"}}
+{"time":"2024-11-05T19:31:02.738902832-08:00","level":"INFO","msg":"handler: started","stream_id":{"value":"jcj67gg8"}}
+{"time":"2024-11-05T19:31:03.056521545-08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-05T23:38:31.946246118-08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-05T23:38:32.023973784-08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-05T23:38:32.922341453-08:00","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"saving job artifact","runtime_seconds":0.775009129}],"total_operations":1}}
+{"time":"2024-11-05T23:38:33.390878425-08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-05T23:38:34.555591357-08:00","level":"INFO","msg":"stream: closing","id":"jcj67gg8"}
+{"time":"2024-11-05T23:38:34.555667186-08:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"jcj67gg8"}}
+{"time":"2024-11-05T23:38:34.555706801-08:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"jcj67gg8"}}
+{"time":"2024-11-05T23:38:34.555872-08:00","level":"INFO","msg":"sender: closed","stream_id":"jcj67gg8"}
+{"time":"2024-11-05T23:38:34.558526671-08:00","level":"INFO","msg":"stream: closed","id":"jcj67gg8"}

wandb/run-20241105_193102-jcj67gg8/logs/debug.log

@@ -0,0 +1,32 @@
+2024-11-05 19:31:02,511 INFO    MainThread:2188020 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2024-11-05 19:31:02,511 INFO    MainThread:2188020 [wandb_setup.py:_flush():79] Configure stats pid to 2188020
+2024-11-05 19:31:02,511 INFO    MainThread:2188020 [wandb_setup.py:_flush():79] Loading settings from /hai/scratch/belkhale/.config/wandb/settings
+2024-11-05 19:31:02,511 INFO    MainThread:2188020 [wandb_setup.py:_flush():79] Loading settings from /hai/scratch/belkhale/openvla-mini/wandb/settings
+2024-11-05 19:31:02,511 INFO    MainThread:2188020 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'_service_wait': '300'}
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/hai/scratch/belkhale/openvla-mini/scripts/pretrain.py', 'program': '/hai/scratch/belkhale/openvla-mini/scripts/pretrain.py'}
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_init.py:_log_setup():534] Logging user logs to runs/prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7/wandb/run-20241105_193102-jcj67gg8/logs/debug.log
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_init.py:_log_setup():535] Logging internal logs to runs/prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7/wandb/run-20241105_193102-jcj67gg8/logs/debug-internal.log
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_init.py:init():621] calling init triggers
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {'model': {'type': 'prism-qwen25-extra-dinosiglip-224px+0_5b', 'model_id': 'prism-qwen25-extra-dinosiglip-224px+0_5b', 'arch_specifier': 'no-align+fused-gelu-mlp', 'vision_backbone_id': 'dinosiglip-vit-so-224px', 'llm_backbone_id': 'qwen25-0_5b-extra', 'image_resize_strategy': 'resize-naive', 'llm_max_length': 32768, 'align_epochs': 1, 'align_max_steps': None, 'align_save_every_n_steps': 10000, 'align_global_batch_size': 96, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 2, 'finetune_max_steps': None, 'finetune_save_every_n_steps': 10000, 'finetune_global_batch_size': 64, 'finetune_per_device_batch_size': 4, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'llava-v15', 'dataset_id': 'llava-v15', 'align_stage_components': ['download/llava-laion-cc-sbu-558k/chat.json', 'download/llava-laion-cc-sbu-558k'], 'finetune_stage_components': ['download/llava-v1.5-instruct/llava_v1_5_mix665k.json', 'download/llava-v1.5-instruct'], 'dataset_root_dir': '/hai/scratch/belkhale/datasets/prismatic-vlms'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'prism-qwen25-extra-dinosiglip-224px+0_5b+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'hf_token': '.hf_token', 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'prismatic', 'wandb_entity': None}
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_init.py:init():671] starting backend
+2024-11-05 19:31:02,512 INFO    MainThread:2188020 [wandb_init.py:init():675] sending inform_init request
+2024-11-05 19:31:02,513 INFO    MainThread:2188020 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-11-05 19:31:02,513 INFO    MainThread:2188020 [wandb_init.py:init():688] backend started and connected
+2024-11-05 19:31:02,515 INFO    MainThread:2188020 [wandb_init.py:init():783] updated telemetry
+2024-11-05 19:31:02,573 INFO    MainThread:2188020 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2024-11-05 19:31:03,050 INFO    MainThread:2188020 [wandb_init.py:init():867] starting run threads in backend
+2024-11-05 19:31:03,226 INFO    MainThread:2188020 [wandb_run.py:_console_start():2463] atexit reg
+2024-11-05 19:31:03,227 INFO    MainThread:2188020 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2024-11-05 19:31:03,227 INFO    MainThread:2188020 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2024-11-05 19:31:03,227 INFO    MainThread:2188020 [wandb_run.py:_redirect():2401] Redirects installed.
+2024-11-05 19:31:03,230 INFO    MainThread:2188020 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-05 23:38:31,920 INFO    MainThread:2188020 [wandb_run.py:_finish():2158] finishing run belkhale/prismatic/jcj67gg8
+2024-11-05 23:38:31,920 INFO    MainThread:2188020 [wandb_run.py:_atexit_cleanup():2426] got exitcode: 0
+2024-11-05 23:38:31,921 INFO    MainThread:2188020 [wandb_run.py:_restore():2408] restore
+2024-11-05 23:38:31,921 INFO    MainThread:2188020 [wandb_run.py:_restore():2414] restore done
+2024-11-05 23:38:34,516 INFO    MainThread:2188020 [wandb_run.py:_footer_history_summary_info():3975] rendering history
+2024-11-05 23:38:34,517 INFO    MainThread:2188020 [wandb_run.py:_footer_history_summary_info():4007] rendering summary
+2024-11-05 23:38:34,534 INFO    MainThread:2188020 [wandb_run.py:_footer_sync_info():3934] logging synced files

wandb/run-20241105_193102-jcj67gg8/run-jcj67gg8.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1e028b995d945fb180c85455d7219269515a28888eda671f635380e4dac0d23
+size 37709802