base_model: codellama/CodeLlama-7b-hf | |
base_model_config: codellama/CodeLlama-7b-hf | |
model_type: LlamaForCausalLM | |
tokenizer_type: CodeLlamaTokenizer | |
is_llama_derived_model: true | |
load_in_8bit: true | |
load_in_4bit: false | |
strict: false | |
datasets: | |
- path: mhenrichsen/alpaca_2k_test | |
type: alpaca | |
dataset_prepared_path: last_run_prepared | |
val_set_size: 0.01 | |
output_dir: ./lora-out | |
sequence_len: 100000 | |
sample_packing: true | |
adapter: lora | |
lora_model_dir: | |
lora_r: 32 | |
lora_alpha: 16 | |
lora_dropout: 0.05 | |
lora_target_linear: true | |
lora_fan_in_fan_out: | |
wandb_project: | |
wandb_entity: | |
wandb_watch: | |
wandb_run_id: | |
wandb_log_model: | |
gradient_accumulation_steps: 4 | |
micro_batch_size: 2 | |
num_epochs: 3 | |
optimizer: adamw_bnb_8bit | |
lr_scheduler: cosine | |
learning_rate: 0.0002 | |
train_on_inputs: false | |
group_by_length: false | |
bf16: true | |
fp16: false | |
tf32: false | |
gradient_checkpointing: true | |
early_stopping_patience: | |
resume_from_checkpoint: | |
local_rank: | |
logging_steps: 1 | |
xformers_attention: | |
flash_attention: true | |
warmup_steps: 10 | |
eval_steps: 20 | |
save_steps: | |
debug: | |
deepspeed: | |
weight_decay: 0.0 | |
fsdp: | |
fsdp_config: | |
special_tokens: | |
bos_token: "<s>" | |
eos_token: "</s>" | |
unk_token: "<unk>" | |