config.yaml

run_name: multitask_train
seed: 6198
epoch: null
dry_run: false
model:
  d_model: 3584
  n_heads: 28
  n_kv_heads: 4
  qkv_bias: true
  clip_qkv: null
  n_layers: 28
  mlp_ratio: 4
  mlp_hidden_size: 37888
  activation_type: swiglu
  block_type: sequential
  block_group_size: 1
  alibi: false
  alibi_bias_max: 8.0
  rope: true
  rope_full_precision: true
  rope_theta: 1000000.0
  rope_impl: llama
  vision_backbone:
    image_model_type: openai
    image_default_input_size:
    - 336
    - 336
    image_patch_size: 14
    image_pos_patch_size: 14
    image_emb_dim: 1024
    image_num_heads: 16
    image_num_key_value_heads: 16
    image_num_layers: 23
    image_head_dim: 64
    image_mlp_dim: 4096
    image_mlp_activations: quick_gelu
    image_dropout_rate: 0.0
    image_num_pos: 577
    image_norm_eps: 1.0e-05
    attention_dropout: 0.0
    residual_dropout: 0.0
    initializer_range: 0.02
    fsdp_wrap: false
    resize_mode: default
  vit_load_path: /weka/oe-training-default/mm-olmo/pretrained_image_encoders/vit-l-14-336.pt
  llm_load_path: /weka/oe-training-default/mm-olmo/pretrained_llms/qwen2-7b.pt
  low_cpu_fsdp: true
  attention_type: sdpa
  float32_attention: true
  attention_dropout: 0.0
  response_attention_dropout: 0.0
  multi_query_attention: null
  attention_layer_norm: false
  residual_dropout: 0.0
  response_residual_dropout: 0.1
  embedding_dropout: 0.0
  layer_norm_type: rms
  layer_norm_with_affine: true
  layer_norm_eps: 1.0e-06
  attention_layer_norm_with_affine: true
  max_sequence_length: 4096
  max_position_embeddings: null
  include_bias: false
  bias_for_layer_norm: null
  scale_logits: false
  vocab_size: 152064
  embedding_size: 152064
  additional_vocab_size: 128
  new_embedding_init_range: 0.02
  weight_tying: false
  pad_token_id: -1
  init_device: null
  init_fn: normal
  init_std: 0.02
  init_cutoff_factor: null
  norm_after: false
  precision: amp_bf16
  max_crops: 12
  crop_mode: overlap-and-resize-c2
  do_random_scale: false
  use_col_tokens: true
  prompt_type: none
  system_prompt_kind: style_and_length
  message_formatting: none
  always_start_with_space: true
  prompt_override: null
  default_inference_len: 65
  overlap_margins:
  - 4
  - 4
  image_padding_embed: pad_and_partial_pad
  vit_layers:
  - -2
  - -9
  image_pooling_h: 2
  image_pooling_w: 2
  image_pooling_2d: attention_meanq
  image_projector: mlp
  image_feature_dropout: 0.0
  use_cls_feature: false
  fix_image_input_idx: 2
  unconditioned: false
  pad_to: null
  initializer_range: 0.02
  pad_tokenizer: true
  normalize_input_embeds: false
  use_position_ids: true
  query_pre_attn_scalar: 224
  attn_logit_softcapping: null
  final_logit_softcapping: null
  head_dim: null
  tokenizer:
    identifier: mm:hf-Qwen/Qwen2-7B
    truncate_direction: right
    tokenizer_adds_space: false
    tokenizer_dir: null
    olmo_bos_token_id: null
    olmo_eos_token_id: null
  loss_token_weighting: null
  gin_bindings: null
ft_llm: true
ft_vit: true
ft_connector: true
ft_embedding: lm_head
optimizer:
  name: adamw
  learning_rate: 0.0001
  weight_decay: 0.01
  betas:
  - 0.9
  - 0.95
  eps: 1.0e-05
  connector_learning_rate: 0.0002
  vit_learning_rate: 6.0e-06
  llm_learning_rate: 2.0e-05
  connector_weight_decay: 0.0
  vit_weight_decay: 0.0
  llm_weight_decay: 0.0
  connector_betas:
  - 0.9
  - 0.95
  vit_betas:
  - 0.9
  - 0.95
  llm_betas:
  - 0.9
  - 0.95
  connector_eps: 1.0e-06
  vit_eps: 1.0e-06
  llm_eps: 1.0e-06
  no_decay_norm_and_bias: null
  decay_norm_and_bias: false
  decay_embeddings: false
  metrics_log_interval: 20
scheduler:
  name: multimodal
  units: steps
  t_warmup: 100
  t_max: null
  alpha_f: 0.1
  connector_t_warmup: 200
  vit_t_warmup: 2000
  llm_t_warmup: 2000
  grad_clip_warmup_steps: null
  grad_clip_warmup_factor: null
  warmup_min_lr: 0.0
data:
  multi_modal: true
  mixture_or_task_name: cockatoo_and_transcript_712k_sept6
  paths: null
  datasets: null
  label_mask_paths: null
  pad_direction: right
  generate_attention_mask: false
  num_workers: 0
  drop_last: true
  pin_memory: false
  prefetch_factor: null
  persistent_workers: false
  timeout: 0
  seed: null
  instance_filter: null
  mixture: null
  sequence_length: 2304
  shuffle: true
  for_inference: false
  split: train
  use_memory_cache: false
  num_epochs: null
  shuffle_buffer_size: 1000
  per_node_data_loader: null
restore_dataloader: true
fast_forward_batches: null
evaluators:
- label: val
  type: multi_modal_lm
  data:
    multi_modal: true
    mixture_or_task_name: cockatoo_and_transcript_712k_sept6
    paths: null
    datasets: null
    label_mask_paths: null
    pad_direction: right
    generate_attention_mask: false
    num_workers: 0
    drop_last: true
    pin_memory: false
    prefetch_factor: null
    persistent_workers: false
    timeout: 0
    seed: null
    instance_filter: null
    mixture: null
    sequence_length: 2304
    shuffle: false
    for_inference: false
    split: validation
    use_memory_cache: false
    num_epochs: null
    shuffle_buffer_size: 1000
    per_node_data_loader: null
  device_eval_batch_size: null
  subset_num_batches: 8
  max_new_tokens: 448
  mm_evaluator: null
  save_dir: null
  save_to_checkpoint_dir: false
  eval_name: null
  skip_if_metrics_cached: true
- label: caption_val
  type: multi_modal_lm
  data:
    multi_modal: true
    mixture_or_task_name: cockatoo_476k_gpt_captions
    paths: null
    datasets: null
    label_mask_paths: null
    pad_direction: right
    generate_attention_mask: false
    num_workers: 0
    drop_last: true
    pin_memory: false
    prefetch_factor: null
    persistent_workers: false
    timeout: 0
    seed: null
    instance_filter: null
    mixture: null
    sequence_length: 2304
    shuffle: false
    for_inference: false
    split: validation
    use_memory_cache: false
    num_epochs: null
    shuffle_buffer_size: 1000
    per_node_data_loader: null
  device_eval_batch_size: null
  subset_num_batches: 8
  max_new_tokens: 448
  mm_evaluator: null
  save_dir: null
  save_to_checkpoint_dir: false
  eval_name: null
  skip_if_metrics_cached: true
eval_interval: 1000
inf_eval_interval: -1
inf_evaluators: []
save_folder: /weka/oe-training-default/chrisc/cockatoo/models/dense-captioner-v22-qwen2/v2-lr2620
remote_save_folder: null
canceled_check_interval: 50
save_interval: 4000
save_interval_unsharded: 22300
save_interval_ephemeral: null
save_num_checkpoints_to_keep: 1
save_num_unsharded_checkpoints_to_keep: -1
save_overwrite: true
force_save_unsharded: false
no_pre_train_checkpoint: true
initial_model_checkpoint: null
load_model_config: null
load_path: null
load_path_sharded_checkpointer: null
reset_optimizer_state: false
reset_trainer_state: false
save_dataloader_state: false
reset_dataloader_state: false
sharded_checkpointer: torch_legacy
new_style_checkpoints: null
max_duration: 22300
global_train_batch_size: 128
device_train_batch_size: 2
device_train_microbatch_size: 4
device_eval_batch_size: 4
eval_subset_num_batches: -1
eval_on_load: false
device_inf_eval_batch_size: 16
inf_eval_subset_num_batches: -1
device_train_grad_accum: 0
max_grad_norm: 1.0
batch_divisor: global_batch
max_grad_norm_ratio: null
precision: amp_bf16
wandb:
  project: cockatoo
  entity: prior-ai2
  group: dense-captioner-v22-qwen2
  name: v2-lr2620
  tags:
  - watching
  log_artifacts: false
  rank_zero_only: true
  log_interval: 20
speed_monitor:
  window_size: 20
  gpu_flops_available: null
console_log_interval: 20
gen1_gc_interval: 1
compile: null
fsdp:
  use_orig_params: true
  sharding_strategy: FULL_SHARD
  wrapping_strategy: by_block_and_size
  precision: float
  hybrid_sharding_num_model_replicas: null
softmax_auxiliary_loss: true
softmax_auxiliary_loss_scale: 0.0001
time_limit: null
extra_steps_after_cancel: 10
early_stopping_factor: null
save_data_indices: false
python_profiling: false
torch_profiling: false
stop_at: 22300
stop_after: null
activation_checkpointing: whole_layer
fused_loss: null
tfds_dir: /weka/oe-training-default/mm-olmo/tensorflow_datasets