9em124t2-499968/config.yaml

wandb_project: joy-caption-1
device_batch_size: 2
batch_size: 256
learning_rate: 0.0002
warmup_samples: 18000
max_samples: 500000
save_every: 50000
test_every: 50000
use_amp: true
grad_scaler: true
lr_scheduler_type: cosine
min_lr_ratio: 0.0
allow_tf32: true
seed: 69
num_workers: 8
optimizer_type: adamw
adam_beta1: 0.9
adam_beta2: 0.999
adam_eps: 1.0e-08
adam_weight_decay: 0.0
clip_grad_norm: 1.0
dataset: fancyfeast/joy-captioning-20240917a
clip_model: google/siglip-so400m-patch14-384
text_model: meta-llama/Meta-Llama-3.1-8B
resume: null
gradient_checkpointing: false
test_size: 2048
grad_scaler_init: 65536.0
max_caption_length: 257
num_image_tokens: 32
adapter_type: mlp
text_model_dtype: bfloat16
pre_test: false
train_image_model: true
image_model_lr: null
train_lora: true
lora_r: 64
lora_alpha: 16
lora_dropout: 0.1