End of training

README.md

@@ -0,0 +1,236 @@
+---
+library_name: transformers
+license: other
+base_model: sbintuitions/sarashina2.1-1b
+tags:
+- axolotl
+- generated_from_trainer
+model-index:
+- name: sarashina2.1-1b-sft
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
+<details><summary>See axolotl config</summary>
+
+axolotl version: `0.5.2`
+```yaml
+base_model: sbintuitions/sarashina2.1-1b
+model_type: AutoModelForCausalLM
+tokenizer_type: AutoTokenizer
+
+hub_model_id: Aratako/sarashina2.1-1b-sft
+hub_strategy: "end"
+push_dataset_to_hub:
+hf_use_auth_token: true
+
+plugins:
+  - axolotl.integrations.liger.LigerPlugin
+liger_cross_entropy: false
+liger_rope: true
+liger_rms_norm: true
+liger_swiglu: true
+liger_fused_linear_cross_entropy: true
+
+load_in_8bit: false
+load_in_4bit: false
+strict: false
+
+chat_template: chatml
+
+datasets:
+  - path: Aratako/Magpie-Tanuki-Qwen2.5-72B-Answered
+    type: chat_template
+    field_messages: messages
+    message_field_role: role
+    message_field_content: content
+  - path: Aratako/magpie-qwen2.5-32b-reasoning-100k-formatted
+    type: chat_template
+    field_messages: conversations
+    message_field_role: role
+    message_field_content: content
+  - path: Aratako/Open-Platypus-Japanese-masked-formatted
+    type: chat_template
+    field_messages: conversations
+    message_field_role: role
+    message_field_content: content
+  - path: kanhatakeyama/wizardlm8x22b-logical-math-coding-sft_additional-ja
+    type: chat_template
+    field_messages: messages
+    message_field_role: role
+    message_field_content: content
+  - path: kanhatakeyama/ramdom-to-fixed-multiturn-Calm3
+    split: 20240806filtered
+    type: chat_template
+    field_messages: messages
+    message_field_role: role
+    message_field_content: content
+  - path: llm-jp/magpie-sft-v1.0
+    type: chat_template
+    field_messages: conversations
+    message_field_role: role
+    message_field_content: content
+  - path: Aratako/aya-ja-evol-instruct-calm3-dpo-masked-sft
+    type: chat_template
+    field_messages: messages
+    message_field_role: role
+    message_field_content: content
+  - path: Aratako/aya-ja-nemotron-dpo-masked-sft
+    type: chat_template
+    field_messages: messages
+    message_field_role: role
+    message_field_content: content
+  - path: Aratako/Synthetic-JP-EN-Coding-Dataset-801k
+    split: "train[0:50000]"
+    type: chat_template
+    field_messages: messages
+    message_field_role: role
+    message_field_content: content
+  - path: Aratako/orca-agentinstruct-1M-v1-selected-2
+    type: chat_template
+    field_messages: messages
+    message_field_role: role
+    message_field_content: content
+  - path: Aratako/Synthetic-JP-EN-Coding-Dataset-801k-50k
+    type: chat_template
+    field_messages: messages
+    message_field_role: role
+    message_field_content: content
+
+shuffle_merged_datasets: true
+dataset_prepared_path: /workspace/data/fft-data-sarashina
+val_set_size: 0.002
+output_dir: /workspace/data/1b-fft-out
+
+sequence_len: 4096
+sample_packing: true
+eval_sample_packing: false
+pad_to_sequence_len: true
+
+adapter:
+lora_model_dir:
+lora_r:
+lora_alpha:
+lora_dropout:
+lora_target_linear:
+lora_fan_in_fan_out:
+
+wandb_project: 1b-fft
+wandb_entity: aratako-lm
+wandb_watch:
+wandb_name: fft-attempt-1
+wandb_log_model:
+
+gradient_accumulation_steps: 4
+micro_batch_size: 8
+num_epochs: 2
+optimizer: adamw_torch
+lr_scheduler: cosine
+cosine_min_lr_ratio: 0.1
+learning_rate: 0.00002
+
+train_on_inputs: false
+group_by_length: false
+bf16: auto
+fp16:
+tf32: false
+
+gradient_checkpointing: false
+early_stopping_patience:
+auto_resume_from_checkpoints: true
+local_rank:
+logging_steps: 1
+xformers_attention:
+flash_attention: true
+
+save_strategy: steps
+save_steps: 100
+save_total_limit: 1
+
+warmup_steps: 20
+eval_steps: 100
+eval_batch_size: 1
+eval_table_size:
+eval_max_new_tokens:
+debug:
+deepspeed: /workspace/axolotl/deepspeed_configs/zero1.json
+weight_decay: 0.01
+fsdp:
+fsdp_config:
+special_tokens:
+  pad_token: <pad>
+
+tokens:
+  - "<|im_start|>"
+  - "<|im_end|>"
+
+```
+
+</details><br>
+
+# sarashina2.1-1b-sft
+
+This model is a fine-tuned version of [sbintuitions/sarashina2.1-1b](https://huggingface.co/sbintuitions/sarashina2.1-1b) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.9366
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 8
+- eval_batch_size: 1
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 256
+- total_eval_batch_size: 8
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_steps: 20
+- num_epochs: 2
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| 1.2935        | 0.0015 | 1    | 1.4733          |
+| 0.985         | 0.1515 | 100  | 1.0491          |
+| 0.9131        | 0.3029 | 200  | 1.0156          |
+| 0.9174        | 0.4544 | 300  | 0.9935          |
+| 0.9257        | 0.6058 | 400  | 0.9806          |
+| 0.869         | 0.7573 | 500  | 0.9694          |
+| 0.8874        | 0.9087 | 600  | 0.9608          |
+| 0.8041        | 1.0594 | 700  | 0.9557          |
+| 0.8348        | 1.2109 | 800  | 0.9512          |
+| 0.8353        | 1.3624 | 900  | 0.9466          |
+| 0.8145        | 1.5138 | 1000 | 0.9432          |
+| 0.8057        | 1.6653 | 1100 | 0.9400          |
+| 0.838         | 1.8167 | 1200 | 0.9381          |
+| 0.8446        | 1.9682 | 1300 | 0.9366          |
+
+
+### Framework versions
+
+- Transformers 4.46.3
+- Pytorch 2.3.1+cu121
+- Datasets 3.1.0
+- Tokenizers 0.20.3

added_tokens.json

@@ -0,0 +1,4 @@
+{
+  "<|im_end|>": 102401,
+  "<|im_start|>": 102400
+}

config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "sbintuitions/sarashina2.1-1b",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 112,
+  "hidden_act": "silu",
+  "hidden_size": 1792,
+  "initializer_range": 0.02,
+  "intermediate_size": 6272,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 500000,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.3",
+  "use_cache": false,
+  "vocab_size": 102402
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "do_sample": true,
+  "eos_token_id": 2,
+  "transformers_version": "4.46.3"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89f6b2fd14403f6eade689340379a88dac517785b60b616bf7543f385594b872
+size 2815117504

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aff9f2c8e35db130a0113609e1d2fb9f1febfd8059457d05bd59fd9d02374e1c
+size 2815122106

special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<cls>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "<sep>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:008293028e1a9d9a1038d9b63d989a2319797dfeaa03f171093a57b33a3a8277
+size 1831879

tokenizer_config.json

@@ -0,0 +1,187 @@
+{
+  "add_bos_token": false,
+  "add_dummy_prefix_space": false,
+  "add_eos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<cls>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<|system|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "8": {
+      "content": "<|assistant|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "9": {
+      "content": "<|user|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "10": {
+      "content": "<|available_tools|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "11": {
+      "content": "<|tool_calls|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "12": {
+      "content": "<|tool_results|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "13": {
+      "content": "<|code|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "14": {
+      "content": "<|file|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "102397": {
+      "content": "<|prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "102398": {
+      "content": "<|suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "102399": {
+      "content": "<|middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "102400": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "102401": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<s>",
+  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "<cls>",
+  "do_lower_case": false,
+  "eos_token": "</s>",
+  "extra_ids": 0,
+  "keep_accents": true,
+  "legacy": false,
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "sep_token": "<sep>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:511211b07bb545451edb5d7ca9e8de45751900bf6d05f5a6b371bd2f198c291e
+size 8120