base_model: tiiuae/falcon-7b | |
base_model_config: tiiuae/falcon-7b | |
trust_remote_code: true | |
model_type: AutoModelForCausalLM | |
tokenizer_type: AutoTokenizer | |
load_in_8bit: false | |
load_in_4bit: false | |
gptq: false | |
strict: false | |
push_dataset_to_hub: | |
datasets: | |
- path: teknium/GPT4-LLM-Cleaned | |
type: alpaca:chat | |
dataset_prepared_path: last_run_prepared | |
val_set_size: 0.01 | |
adapter: | |
lora_model_dir: | |
sequence_len: 2048 | |
max_packed_sequence_len: | |
lora_r: 64 | |
lora_alpha: 32 | |
lora_dropout: 0.0 | |
lora_target_modules: | |
lora_target_linear: true | |
lora_fan_in_fan_out: | |
wandb_project: falcon-7b | |
wandb_watch: | |
wandb_run_id: | |
wandb_log_model: | |
output_dir: ./falcon-7b | |
batch_size: 2 | |
micro_batch_size: 1 | |
num_epochs: 4 | |
optimizer: adamw_bnb_8bit | |
torchdistx_path: | |
lr_scheduler: cosine | |
learning_rate: 0.00003 | |
train_on_inputs: false | |
group_by_length: false | |
bf16: true | |
fp16: false | |
tf32: true | |
gradient_checkpointing: true | |
early_stopping_patience: | |
resume_from_checkpoint: | |
local_rank: | |
logging_steps: 1 | |
xformers_attention: true | |
flash_attention: | |
gptq_groupsize: | |
gptq_model_v1: | |
warmup_steps: 40 | |
eval_steps: 5 | |
save_steps: 43 | |
debug: | |
deepspeed: | |
weight_decay: 0.0 | |
fsdp: | |
fsdp_config: | |
special_tokens: | |
pad_token: "<|endoftext|>" | |
bos_token: ">>ABSTRACT<<" | |
eos_token: "<|endoftext|>" | |