Update README.md
Browse files
README.md
CHANGED
@@ -1,6 +1,7 @@
|
|
1 |
# This repo contains EleutherAI/gpt-j-6B fine tuned on OWCA (https://github.com/Emplocity/owca) using LoRa
|
2 |
|
3 |
Training params:<br/>
|
|
|
4 |
MICRO_BATCH_SIZE = 64
|
5 |
BATCH_SIZE = 128
|
6 |
GRADIENT_ACCUMULATION_STEPS = BATCH_SIZE // MICRO_BATCH_SIZE
|
@@ -11,4 +12,5 @@ LORA_R = 4
|
|
11 |
LORA_ALPHA = 16
|
12 |
LORA_DROPOUT = 0.05
|
13 |
warmup_steps=100
|
14 |
-
fp16=True
|
|
|
|
1 |
# This repo contains EleutherAI/gpt-j-6B fine tuned on OWCA (https://github.com/Emplocity/owca) using LoRa
|
2 |
|
3 |
Training params:<br/>
|
4 |
+
```
|
5 |
MICRO_BATCH_SIZE = 64
|
6 |
BATCH_SIZE = 128
|
7 |
GRADIENT_ACCUMULATION_STEPS = BATCH_SIZE // MICRO_BATCH_SIZE
|
|
|
12 |
LORA_ALPHA = 16
|
13 |
LORA_DROPOUT = 0.05
|
14 |
warmup_steps=100
|
15 |
+
fp16=True
|
16 |
+
```
|