Update README.md

README.md

@@ -1,3 +1,63 @@
 ---
 license: other
 ---
+
+- [wand](https://wandb.ai/open-assistant/supervised-finetuning/runs/2jfazjt9) (still internal, needs to be moved to public-sft)
+- checkpoint: 3319 steps
+
+## Note
+
+In order to load this model you need to install a pre-release version of the Huggingface transformers library.
+
+
+## Model Configuration
+```
+llama2_13b_orca_8k:
+  rng_seed: 0xe1291f1a
+  use_custom_sampler: true
+  sort_by_length: false
+  dtype: fp16
+  log_dir: "llama2_log_13b_orca_8k"
+  learning_rate: 1e-5
+  model_name: /mnt/data/llama2/Llama-2-13b-hf/
+  output_dir: llama2_13b_orca_8k
+  deepspeed_config: configs/zero_config_pretrain.json
+  weight_decay: 0.0
+  max_length: 8192
+  warmup_steps: 100
+  use_flash_attention: true
+  gradient_checkpointing: true
+  gradient_accumulation_steps: 8
+  per_device_train_batch_size: 2
+  per_device_eval_batch_size: 1
+  residual_dropout: 0.0
+  eval_steps: 200
+  save_steps: 1000  # (total steps: 3319)
+  num_train_epochs: 1
+  save_total_limit: 4
+  superhot: true
+  superhot_config:
+    type: linear
+    scale: 2
+  datasets:
+    # Dataset Composition:
+    # Tain (sampled):
+    #   orca-chat: 100.00% (188842)
+    #   fanfics: 100.00% (47760)
+    #   red_pajama: 25.00% (188262)
+    # Valid:
+    #   orca-chat: 5000 (71.43%)
+    #   fanfics: 1000 (14.29%)
+    #   red_pajama: 1000 (14.29%)
+    - orca-chat:
+        max_val_set: 5000
+    - fanfics:
+        max_chunk_size: 65535
+        max_val_set: 1000
+    - red_pajama:
+        fraction: 0.25
+        max_val_set: 1000
+        max_chunk_size: 65535
+  peft_model: false
+```
+