huggingartists

README.md

@@ -45,15 +45,15 @@ from datasets import load_dataset
 dataset = load_dataset("huggingartists/morgenshtern")
 ```
 
-[Explore the data](https://wandb.ai/huggingartists/huggingartists/runs/1qveqjla/artifacts), which is tracked with [W&B artifacts](https://docs.wandb.com/artifacts) at every step of the pipeline.
+[Explore the data](https://wandb.ai/huggingartists/huggingartists/runs/29htvpbu/artifacts), which is tracked with [W&B artifacts](https://docs.wandb.com/artifacts) at every step of the pipeline.
 
 ## Training procedure
 
 The model is based on a pre-trained [GPT-2](https://huggingface.co/gpt2) which is fine-tuned on MORGENSHTERN's lyrics.
 
-Hyperparameters and metrics are recorded in the [W&B training run](https://wandb.ai/huggingartists/huggingartists/runs/21tohu9l) for full transparency and reproducibility.
+Hyperparameters and metrics are recorded in the [W&B training run](https://wandb.ai/huggingartists/huggingartists/runs/m6tldjdu) for full transparency and reproducibility.
 
-At the end of training, [the final model](https://wandb.ai/huggingartists/huggingartists/runs/21tohu9l/artifacts) is logged and versioned.
+At the end of training, [the final model](https://wandb.ai/huggingartists/huggingartists/runs/m6tldjdu/artifacts) is logged and versioned.
 
 ## How to use
 

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "huggingartists/morgenshtern",
+  "_name_or_path": "morgenshtern",
   "activation_function": "gelu_new",
   "architectures": [
     "GPT2LMHeadModel"
@@ -18,7 +18,9 @@
   "n_inner": null,
   "n_layer": 12,
   "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
   "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
   "scale_attn_weights": true,
   "summary_activation": null,
   "summary_first_dropout": 0.1,
@@ -35,7 +37,7 @@
     }
   },
   "torch_dtype": "float32",
-  "transformers_version": "4.11.3",
+  "transformers_version": "4.16.2",
   "use_cache": true,
   "vocab_size": 50257
 }

evaluation.txt

@@ -1 +1 @@
-{"eval_loss": 0.8691701889038086, "eval_runtime": 5.5874, "eval_samples_per_second": 22.014, "eval_steps_per_second": 2.864, "epoch": 4.0}
+{"eval_loss": 0.6951560974121094, "eval_runtime": 6.9424, "eval_samples_per_second": 20.598, "eval_steps_per_second": 2.593, "epoch": 3.0}

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0dcb9fd0c95e49a7c5c6a915f2d7adaa973be6d1ae19ff9baad7139b0524ce2b
+oid sha256:bd5ea8451fb818bbbf19558ab096fa4f81596c541e2622568412ebf5daf0efd2
 size 497764120

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b9057c2e701481d1224ab71cb3e30e9a63129f8298087d242e9dbef4d013a3c
+oid sha256:e4755f815eb61413dfd1b66a9a03a2b2eb842a2b33d9bff4a55141c7a91df9bf
 size 995604017

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fff17a1d5c7cb31c35913396acbea4856780b0256c64b1a8d4aaee9270629fc7
+oid sha256:0dc86c1d0e5ed8e708b41cbb9b119c5960c259dba079c2cee589bb9afa77c5ca
 size 510403817

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a9ebc7a18dd93d6b68e430a5a307587231b2cf1f61199f2bfb8080060d317195
+oid sha256:8dfdfe168c690fdd51fc3442227da71ee0a9fd24f3602c5faf62e0fb7e6e7755
 size 14503

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b46773a8e0da9899f0332e86140882baf8db0e4d332cd8d2b17d122dbb838867
+oid sha256:f889ee048c975721acd46c969685ee09ff3f7ad66edbe67905c8a29f4eede81b
 size 623

trainer_state.json

@@ -1,8 +1,8 @@
 {
-  "best_metric": 0.8691701889038086,
-  "best_model_checkpoint": "output/morgenshtern/checkpoint-202",
-  "epoch": 2.0,
-  "global_step": 202,
+  "best_metric": 0.6951560974121094,
+  "best_model_checkpoint": "output/morgenshtern/checkpoint-294",
+  "epoch": 3.0,
+  "global_step": 294,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -262,11 +262,127 @@
       "eval_samples_per_second": 22.717,
       "eval_steps_per_second": 2.955,
       "step": 202
+    },
+    {
+      "epoch": 2.09,
+      "learning_rate": 0.00013663679494827462,
+      "loss": 0.8056,
+      "step": 205
+    },
+    {
+      "epoch": 2.14,
+      "learning_rate": 0.00013436461471831498,
+      "loss": 0.8153,
+      "step": 210
+    },
+    {
+      "epoch": 2.19,
+      "learning_rate": 0.00013040646433810593,
+      "loss": 0.8439,
+      "step": 215
+    },
+    {
+      "epoch": 2.24,
+      "learning_rate": 0.00012486381666535154,
+      "loss": 0.8468,
+      "step": 220
+    },
+    {
+      "epoch": 2.3,
+      "learning_rate": 0.00011787876541670438,
+      "loss": 0.9434,
+      "step": 225
+    },
+    {
+      "epoch": 2.35,
+      "learning_rate": 0.00010963038239169752,
+      "loss": 0.9117,
+      "step": 230
+    },
+    {
+      "epoch": 2.4,
+      "learning_rate": 0.0001003301267105212,
+      "loss": 0.9685,
+      "step": 235
+    },
+    {
+      "epoch": 2.45,
+      "learning_rate": 9.021642375642094e-05,
+      "loss": 0.8183,
+      "step": 240
+    },
+    {
+      "epoch": 2.5,
+      "learning_rate": 7.954855279929014e-05,
+      "loss": 0.8455,
+      "step": 245
+    },
+    {
+      "epoch": 2.55,
+      "learning_rate": 6.860000000000005e-05,
+      "loss": 0.8961,
+      "step": 250
+    },
+    {
+      "epoch": 2.6,
+      "learning_rate": 5.765144720071096e-05,
+      "loss": 0.9542,
+      "step": 255
+    },
+    {
+      "epoch": 2.65,
+      "learning_rate": 4.698357624357919e-05,
+      "loss": 0.9393,
+      "step": 260
+    },
+    {
+      "epoch": 2.7,
+      "learning_rate": 3.686987328947893e-05,
+      "loss": 0.9105,
+      "step": 265
+    },
+    {
+      "epoch": 2.76,
+      "learning_rate": 2.7569617608302584e-05,
+      "loss": 0.9638,
+      "step": 270
+    },
+    {
+      "epoch": 2.81,
+      "learning_rate": 1.932123458329638e-05,
+      "loss": 0.8269,
+      "step": 275
+    },
+    {
+      "epoch": 2.86,
+      "learning_rate": 1.233618333464853e-05,
+      "loss": 0.9693,
+      "step": 280
+    },
+    {
+      "epoch": 2.91,
+      "learning_rate": 6.7935356618941304e-06,
+      "loss": 0.8474,
+      "step": 285
+    },
+    {
+      "epoch": 2.96,
+      "learning_rate": 2.8353852816850615e-06,
+      "loss": 0.8138,
+      "step": 290
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.6951560974121094,
+      "eval_runtime": 6.9159,
+      "eval_samples_per_second": 20.677,
+      "eval_steps_per_second": 2.603,
+      "step": 294
     }
   ],
-  "max_steps": 404,
-  "num_train_epochs": 4,
-  "total_flos": 209817501696000.0,
+  "max_steps": 294,
+  "num_train_epochs": 3,
+  "total_flos": 305711677440000.0,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dd3bfc9cf6ae7969e054c62bceff3f5cf23d65ecf03aee4de9a1c7b331576fd6
-size 2863
+oid sha256:e1e43acc9100bdd682eafe8ab7a4ccf0fb744b6214b26c1a24e8e39367a3d089
+size 3055