File size: 872 Bytes
394e8ba
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
wandb_project: joy-caption-1
device_batch_size: 2
batch_size: 256
learning_rate: 0.0002
warmup_samples: 18000
max_samples: 600000
save_every: 50000
test_every: 50000
use_amp: true
grad_scaler: true
lr_scheduler_type: cosine
min_lr_ratio: 0.0
allow_tf32: true
seed: 69
num_workers: 8
optimizer_type: adamw
adam_beta1: 0.9
adam_beta2: 0.999
adam_eps: 1.0e-08
adam_weight_decay: 0.0
clip_grad_norm: 1.0
dataset: fancyfeast/joy-captioning-20240924a
clip_model: google/siglip-so400m-patch14-384
text_model: ../lora-train/lora_model_vwbzycxh
resume: null
gradient_checkpointing: false
test_size: 2048
grad_scaler_init: 65536.0
max_caption_length: 257
num_image_tokens: 32
adapter_type: mlp
text_model_dtype: bfloat16
pre_test: false
train_image_model: true
image_model_lr: null
train_lora: true
lora_r: 64
lora_alpha: 16
lora_dropout: 0.1