| run_name: multitask_train | |
| seed: 6198 | |
| epoch: null | |
| dry_run: false | |
| model: | |
| d_model: 3584 | |
| n_heads: 28 | |
| n_kv_heads: 4 | |
| qkv_bias: true | |
| clip_qkv: null | |
| n_layers: 28 | |
| mlp_ratio: 4 | |
| mlp_hidden_size: 37888 | |
| activation_type: swiglu | |
| block_type: sequential | |
| block_group_size: 1 | |
| alibi: false | |
| alibi_bias_max: 8.0 | |
| rope: true | |
| rope_full_precision: true | |
| rope_theta: 1000000.0 | |
| rope_impl: llama | |
| vision_backbone: | |
| image_model_type: openai | |
| image_default_input_size: | |
| - 336 | |
| - 336 | |
| image_patch_size: 14 | |
| image_pos_patch_size: 14 | |
| image_emb_dim: 1024 | |
| image_num_heads: 16 | |
| image_num_key_value_heads: 16 | |
| image_num_layers: 23 | |
| image_head_dim: 64 | |
| image_mlp_dim: 4096 | |
| image_mlp_activations: quick_gelu | |
| image_dropout_rate: 0.0 | |
| image_num_pos: 577 | |
| image_norm_eps: 1.0e-05 | |
| attention_dropout: 0.0 | |
| residual_dropout: 0.0 | |
| initializer_range: 0.02 | |
| fsdp_wrap: false | |
| resize_mode: default | |
| vit_load_path: /weka/oe-training-default/mm-olmo/pretrained_image_encoders/vit-l-14-336.pt | |
| llm_load_path: /weka/oe-training-default/mm-olmo/pretrained_llms/qwen2-7b.pt | |
| low_cpu_fsdp: true | |
| attention_type: sdpa | |
| float32_attention: true | |
| attention_dropout: 0.0 | |
| response_attention_dropout: 0.0 | |
| multi_query_attention: null | |
| attention_layer_norm: false | |
| residual_dropout: 0.0 | |
| response_residual_dropout: 0.1 | |
| embedding_dropout: 0.0 | |
| layer_norm_type: rms | |
| layer_norm_with_affine: true | |
| layer_norm_eps: 1.0e-06 | |
| attention_layer_norm_with_affine: true | |
| max_sequence_length: 4096 | |
| max_position_embeddings: null | |
| include_bias: false | |
| bias_for_layer_norm: null | |
| scale_logits: false | |
| vocab_size: 152064 | |
| embedding_size: 152064 | |
| additional_vocab_size: 128 | |
| new_embedding_init_range: 0.02 | |
| weight_tying: false | |
| pad_token_id: -1 | |
| init_device: null | |
| init_fn: normal | |
| init_std: 0.02 | |
| init_cutoff_factor: null | |
| norm_after: false | |
| precision: amp_bf16 | |
| max_crops: 12 | |
| crop_mode: overlap-and-resize-c2 | |
| do_random_scale: false | |
| use_col_tokens: true | |
| prompt_type: none | |
| system_prompt_kind: style_and_length | |
| message_formatting: none | |
| always_start_with_space: true | |
| prompt_override: null | |
| default_inference_len: 65 | |
| overlap_margins: | |
| - 4 | |
| - 4 | |
| image_padding_embed: pad_and_partial_pad | |
| vit_layers: | |
| - -2 | |
| - -9 | |
| image_pooling_h: 2 | |
| image_pooling_w: 2 | |
| image_pooling_2d: attention_meanq | |
| image_projector: mlp | |
| image_feature_dropout: 0.0 | |
| use_cls_feature: false | |
| fix_image_input_idx: 2 | |
| unconditioned: false | |
| pad_to: null | |
| initializer_range: 0.02 | |
| pad_tokenizer: true | |
| normalize_input_embeds: false | |
| use_position_ids: true | |
| query_pre_attn_scalar: 224 | |
| attn_logit_softcapping: null | |
| final_logit_softcapping: null | |
| head_dim: null | |
| tokenizer: | |
| identifier: mm:hf-Qwen/Qwen2-7B | |
| truncate_direction: right | |
| tokenizer_adds_space: false | |
| tokenizer_dir: null | |
| olmo_bos_token_id: null | |
| olmo_eos_token_id: null | |
| loss_token_weighting: null | |
| gin_bindings: null | |
| ft_llm: true | |
| ft_vit: true | |
| ft_connector: true | |
| ft_embedding: lm_head | |
| optimizer: | |
| name: adamw | |
| learning_rate: 0.0001 | |
| weight_decay: 0.01 | |
| betas: | |
| - 0.9 | |
| - 0.95 | |
| eps: 1.0e-05 | |
| connector_learning_rate: 0.0002 | |
| vit_learning_rate: 6.0e-06 | |
| llm_learning_rate: 2.0e-05 | |
| connector_weight_decay: 0.0 | |
| vit_weight_decay: 0.0 | |
| llm_weight_decay: 0.0 | |
| connector_betas: | |
| - 0.9 | |
| - 0.95 | |
| vit_betas: | |
| - 0.9 | |
| - 0.95 | |
| llm_betas: | |
| - 0.9 | |
| - 0.95 | |
| connector_eps: 1.0e-06 | |
| vit_eps: 1.0e-06 | |
| llm_eps: 1.0e-06 | |
| no_decay_norm_and_bias: null | |
| decay_norm_and_bias: false | |
| decay_embeddings: false | |
| metrics_log_interval: 20 | |
| scheduler: | |
| name: multimodal | |
| units: steps | |
| t_warmup: 100 | |
| t_max: null | |
| alpha_f: 0.1 | |
| connector_t_warmup: 200 | |
| vit_t_warmup: 2000 | |
| llm_t_warmup: 2000 | |
| grad_clip_warmup_steps: null | |
| grad_clip_warmup_factor: null | |
| warmup_min_lr: 0.0 | |
| data: | |
| multi_modal: true | |
| mixture_or_task_name: cockatoo_and_transcript_712k_sept6 | |
| paths: null | |
| datasets: null | |
| label_mask_paths: null | |
| pad_direction: right | |
| generate_attention_mask: false | |
| num_workers: 0 | |
| drop_last: true | |
| pin_memory: false | |
| prefetch_factor: null | |
| persistent_workers: false | |
| timeout: 0 | |
| seed: null | |
| instance_filter: null | |
| mixture: null | |
| sequence_length: 2304 | |
| shuffle: true | |
| for_inference: false | |
| split: train | |
| use_memory_cache: false | |
| num_epochs: null | |
| shuffle_buffer_size: 1000 | |
| per_node_data_loader: null | |
| restore_dataloader: true | |
| fast_forward_batches: null | |
| evaluators: | |
| - label: val | |
| type: multi_modal_lm | |
| data: | |
| multi_modal: true | |
| mixture_or_task_name: cockatoo_and_transcript_712k_sept6 | |
| paths: null | |
| datasets: null | |
| label_mask_paths: null | |
| pad_direction: right | |
| generate_attention_mask: false | |
| num_workers: 0 | |
| drop_last: true | |
| pin_memory: false | |
| prefetch_factor: null | |
| persistent_workers: false | |
| timeout: 0 | |
| seed: null | |
| instance_filter: null | |
| mixture: null | |
| sequence_length: 2304 | |
| shuffle: false | |
| for_inference: false | |
| split: validation | |
| use_memory_cache: false | |
| num_epochs: null | |
| shuffle_buffer_size: 1000 | |
| per_node_data_loader: null | |
| device_eval_batch_size: null | |
| subset_num_batches: 8 | |
| max_new_tokens: 448 | |
| mm_evaluator: null | |
| save_dir: null | |
| save_to_checkpoint_dir: false | |
| eval_name: null | |
| skip_if_metrics_cached: true | |
| - label: caption_val | |
| type: multi_modal_lm | |
| data: | |
| multi_modal: true | |
| mixture_or_task_name: cockatoo_476k_gpt_captions | |
| paths: null | |
| datasets: null | |
| label_mask_paths: null | |
| pad_direction: right | |
| generate_attention_mask: false | |
| num_workers: 0 | |
| drop_last: true | |
| pin_memory: false | |
| prefetch_factor: null | |
| persistent_workers: false | |
| timeout: 0 | |
| seed: null | |
| instance_filter: null | |
| mixture: null | |
| sequence_length: 2304 | |
| shuffle: false | |
| for_inference: false | |
| split: validation | |
| use_memory_cache: false | |
| num_epochs: null | |
| shuffle_buffer_size: 1000 | |
| per_node_data_loader: null | |
| device_eval_batch_size: null | |
| subset_num_batches: 8 | |
| max_new_tokens: 448 | |
| mm_evaluator: null | |
| save_dir: null | |
| save_to_checkpoint_dir: false | |
| eval_name: null | |
| skip_if_metrics_cached: true | |
| eval_interval: 1000 | |
| inf_eval_interval: -1 | |
| inf_evaluators: [] | |
| save_folder: /weka/oe-training-default/chrisc/cockatoo/models/dense-captioner-v22-qwen2/v2-lr2620 | |
| remote_save_folder: null | |
| canceled_check_interval: 50 | |
| save_interval: 4000 | |
| save_interval_unsharded: 22300 | |
| save_interval_ephemeral: null | |
| save_num_checkpoints_to_keep: 1 | |
| save_num_unsharded_checkpoints_to_keep: -1 | |
| save_overwrite: true | |
| force_save_unsharded: false | |
| no_pre_train_checkpoint: true | |
| initial_model_checkpoint: null | |
| load_model_config: null | |
| load_path: null | |
| load_path_sharded_checkpointer: null | |
| reset_optimizer_state: false | |
| reset_trainer_state: false | |
| save_dataloader_state: false | |
| reset_dataloader_state: false | |
| sharded_checkpointer: torch_legacy | |
| new_style_checkpoints: null | |
| max_duration: 22300 | |
| global_train_batch_size: 128 | |
| device_train_batch_size: 2 | |
| device_train_microbatch_size: 4 | |
| device_eval_batch_size: 4 | |
| eval_subset_num_batches: -1 | |
| eval_on_load: false | |
| device_inf_eval_batch_size: 16 | |
| inf_eval_subset_num_batches: -1 | |
| device_train_grad_accum: 0 | |
| max_grad_norm: 1.0 | |
| batch_divisor: global_batch | |
| max_grad_norm_ratio: null | |
| precision: amp_bf16 | |
| wandb: | |
| project: cockatoo | |
| entity: prior-ai2 | |
| group: dense-captioner-v22-qwen2 | |
| name: v2-lr2620 | |
| tags: | |
| - watching | |
| log_artifacts: false | |
| rank_zero_only: true | |
| log_interval: 20 | |
| speed_monitor: | |
| window_size: 20 | |
| gpu_flops_available: null | |
| console_log_interval: 20 | |
| gen1_gc_interval: 1 | |
| compile: null | |
| fsdp: | |
| use_orig_params: true | |
| sharding_strategy: FULL_SHARD | |
| wrapping_strategy: by_block_and_size | |
| precision: float | |
| hybrid_sharding_num_model_replicas: null | |
| softmax_auxiliary_loss: true | |
| softmax_auxiliary_loss_scale: 0.0001 | |
| time_limit: null | |
| extra_steps_after_cancel: 10 | |
| early_stopping_factor: null | |
| save_data_indices: false | |
| python_profiling: false | |
| torch_profiling: false | |
| stop_at: 22300 | |
| stop_after: null | |
| activation_checkpointing: whole_layer | |
| fused_loss: null | |
| tfds_dir: /weka/oe-training-default/mm-olmo/tensorflow_datasets | |