base_model: codellama/CodeLlama-13b-hf | |
model_type: LlamaForCausalLM | |
tokenizer_type: CodeLlamaTokenizer | |
is_llama_derived_model: true | |
load_in_8bit: false | |
load_in_4bit: true | |
strict: false | |
datasets: | |
- path: mhenrichsen/alpaca_2k_test | |
type: alpaca | |
dataset_prepared_path: | |
val_set_size: 0.05 | |
output_dir: ./qlora-out | |
adapter: qlora | |
lora_model_dir: | |
sequence_len: 4096 | |
sample_packing: true | |
pad_to_sequence_len: true | |
lora_r: 32 | |
lora_alpha: 16 | |
lora_dropout: 0.05 | |
lora_target_modules: | |
lora_target_linear: true | |
lora_fan_in_fan_out: | |
wandb_project: | |
wandb_entity: | |
wandb_watch: | |
wandb_name: | |
wandb_log_model: | |
gradient_accumulation_steps: 4 | |
micro_batch_size: 2 | |
num_epochs: 4 | |
optimizer: paged_adamw_32bit | |
lr_scheduler: cosine | |
learning_rate: 0.0002 | |
train_on_inputs: false | |
group_by_length: false | |
bf16: auto | |
fp16: | |
tf32: false | |
gradient_checkpointing: true | |
early_stopping_patience: | |
resume_from_checkpoint: | |
local_rank: | |
logging_steps: 1 | |
xformers_attention: | |
flash_attention: true | |
warmup_steps: 10 | |
evals_per_epoch: 4 | |
saves_per_epoch: 1 | |
debug: | |
deepspeed: | |
weight_decay: 0.0 | |
fsdp: | |
fsdp_config: | |
special_tokens: | |
bos_token: "<s>" | |
eos_token: "</s>" | |
unk_token: "<unk>" | |