Hugging Face's logo

0xsuid
/
GPT-Neo-125M

TensorBoard
Model card Files Metrics Community
0xsuid commited on
Commit
14adc83
·
1 Parent(s): 1f75dd3

Upload experiments/2022-12-21-a12ddf66d95706b1f5f35892f7f60ad628d199c701a09dddd2ad64e8a86a0bae

Browse files
Files changed (8) hide show
  1. experiments/2022-12-21-a12ddf66d95706b1f5f35892f7f60ad628d199c701a09dddd2ad64e8a86a0bae/configs.json +25 -0
  2. experiments/2022-12-21-a12ddf66d95706b1f5f35892f7f60ad628d199c701a09dddd2ad64e8a86a0bae/deepspeed.json +48 -0
  3. experiments/2022-12-21-a12ddf66d95706b1f5f35892f7f60ad628d199c701a09dddd2ad64e8a86a0bae/final_checkpoint/config.json +54 -0
  4. experiments/2022-12-21-a12ddf66d95706b1f5f35892f7f60ad628d199c701a09dddd2ad64e8a86a0bae/final_checkpoint/pytorch_model.bin +3 -0
  5. experiments/2022-12-21-a12ddf66d95706b1f5f35892f7f60ad628d199c701a09dddd2ad64e8a86a0bae/logs/1671556838.4800577/events.out.tfevents.1671556838.2a17441b-6300-46db-8932-ac62cda77916.2267.2 +3 -0
  6. experiments/2022-12-21-a12ddf66d95706b1f5f35892f7f60ad628d199c701a09dddd2ad64e8a86a0bae/logs/events.out.tfevents.1671556838.2a17441b-6300-46db-8932-ac62cda77916.2267.1 +3 -0
  7. experiments/2022-12-21-a12ddf66d95706b1f5f35892f7f60ad628d199c701a09dddd2ad64e8a86a0bae/logs/train_neo/events.out.tfevents.1671556765.2a17441b-6300-46db-8932-ac62cda77916.2267.0 +3 -0
  8. experiments/2022-12-21-a12ddf66d95706b1f5f35892f7f60ad628d199c701a09dddd2ad64e8a86a0bae/output.log +0 -0
experiments/2022-12-21-a12ddf66d95706b1f5f35892f7f60ad628d199c701a09dddd2ad64e8a86a0bae/configs.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "output_dir": "./results",
3
+ "evaluation_strategy": "no",
4
+ "do_eval": false,
5
+ "eval_steps": 0,
6
+ "log_level": "error",
7
+ "logging_first_step": true,
8
+ "logging_steps": 5,
9
+ "logging_dir": "./logs",
10
+ "save_steps": 200,
11
+ "save_total_limit": 2,
12
+ "num_train_epochs": 5,
13
+ "per_device_train_batch_size": 6,
14
+ "optim": "adamw_torch",
15
+ "dataloader_drop_last": true,
16
+ "warmup_steps": 500,
17
+ "weight_decay": 0.05,
18
+ "learning_rate": 5e-05,
19
+ "local_rank": 3,
20
+ "deepspeed": "deepspeed.json",
21
+ "total_gpus": 4,
22
+ "v_cpus": 12,
23
+ "total_memory_in_gb": 48191.34375,
24
+ "dataset_limit": 0
25
+ }
experiments/2022-12-21-a12ddf66d95706b1f5f35892f7f60ad628d199c701a09dddd2ad64e8a86a0bae/deepspeed.json ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "optimizer": {
3
+ "type": "AdamW",
4
+ "params": {
5
+ "lr": "auto",
6
+ "betas": "auto",
7
+ "eps": "auto",
8
+ "weight_decay": "auto"
9
+ }
10
+ },
11
+
12
+ "scheduler": {
13
+ "type": "WarmupLR",
14
+ "params": {
15
+ "warmup_min_lr": "auto",
16
+ "warmup_max_lr": "auto",
17
+ "warmup_num_steps": "auto"
18
+ }
19
+ },
20
+
21
+ "zero_optimization": {
22
+ "stage": 2,
23
+ "offload_optimizer": {
24
+ "device": "cpu",
25
+ "pin_memory": true
26
+ },
27
+ "allgather_partitions": true,
28
+ "allgather_bucket_size": 5e8,
29
+ "overlap_comm": true,
30
+ "reduce_scatter": true,
31
+ "reduce_bucket_size": 5e8,
32
+ "contiguous_gradients": true
33
+ },
34
+
35
+ "tensorboard": {
36
+ "enabled": true,
37
+ "output_path": "logs/",
38
+ "job_name": "train_neo"
39
+ },
40
+
41
+ "zero_allow_untested_optimizer": true,
42
+ "gradient_accumulation_steps": "auto",
43
+ "gradient_clipping": "auto",
44
+ "steps_per_print": 2000,
45
+ "train_batch_size": "auto",
46
+ "train_micro_batch_size_per_gpu": "auto",
47
+ "wall_clock_breakdown": false
48
+ }
experiments/2022-12-21-a12ddf66d95706b1f5f35892f7f60ad628d199c701a09dddd2ad64e8a86a0bae/final_checkpoint/config.json ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "EleutherAI/gpt-neo-125M",
3
+ "activation_function": "gelu_new",
4
+ "architectures": [
5
+ "GPTNeoForCausalLM"
6
+ ],
7
+ "attention_dropout": 0,
8
+ "attention_layers": [
9
+ "global",
10
+ "local",
11
+ "global",
12
+ "local",
13
+ "global",
14
+ "local",
15
+ "global",
16
+ "local",
17
+ "global",
18
+ "local",
19
+ "global",
20
+ "local"
21
+ ],
22
+ "attention_types": [
23
+ [
24
+ [
25
+ "global",
26
+ "local"
27
+ ],
28
+ 6
29
+ ]
30
+ ],
31
+ "bos_token_id": 50256,
32
+ "embed_dropout": 0,
33
+ "eos_token_id": 50256,
34
+ "gradient_checkpointing": false,
35
+ "hidden_size": 768,
36
+ "initializer_range": 0.02,
37
+ "intermediate_size": null,
38
+ "layer_norm_epsilon": 1e-05,
39
+ "max_position_embeddings": 2048,
40
+ "model_type": "gpt_neo",
41
+ "num_heads": 12,
42
+ "num_layers": 12,
43
+ "resid_dropout": 0,
44
+ "summary_activation": null,
45
+ "summary_first_dropout": 0.1,
46
+ "summary_proj_to_labels": true,
47
+ "summary_type": "cls_index",
48
+ "summary_use_proj": true,
49
+ "torch_dtype": "float32",
50
+ "transformers_version": "4.25.1",
51
+ "use_cache": true,
52
+ "vocab_size": 50257,
53
+ "window_size": 256
54
+ }
experiments/2022-12-21-a12ddf66d95706b1f5f35892f7f60ad628d199c701a09dddd2ad64e8a86a0bae/final_checkpoint/pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d1dae543ecfea8918caadf861f881236794f36006b1ff845b0cc89202ef7f842
3
+ size 551154684
experiments/2022-12-21-a12ddf66d95706b1f5f35892f7f60ad628d199c701a09dddd2ad64e8a86a0bae/logs/1671556838.4800577/events.out.tfevents.1671556838.2a17441b-6300-46db-8932-ac62cda77916.2267.2 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:93a65b0b8e4d89bc617e485fdea0539d3561c182f50e87046a5befd661cc2401
3
+ size 5457
experiments/2022-12-21-a12ddf66d95706b1f5f35892f7f60ad628d199c701a09dddd2ad64e8a86a0bae/logs/events.out.tfevents.1671556838.2a17441b-6300-46db-8932-ac62cda77916.2267.1 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0c2e20f24bfe076a8b2a2045af9f5fe2140b6619eb6e61e99aec749cf3be5ab5
3
+ size 776121
experiments/2022-12-21-a12ddf66d95706b1f5f35892f7f60ad628d199c701a09dddd2ad64e8a86a0bae/logs/train_neo/events.out.tfevents.1671556765.2a17441b-6300-46db-8932-ac62cda77916.2267.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ec0dc137250bf95b5f973bd58c5ae55062c10e7eb18e25e40696942a2e7775c1
3
+ size 2929062
experiments/2022-12-21-a12ddf66d95706b1f5f35892f7f60ad628d199c701a09dddd2ad64e8a86a0bae/output.log ADDED
The diff for this file is too large to render. See raw diff