diff --git "a/checkpoint-2535/trainer_state.json" "b/checkpoint-2535/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/checkpoint-2535/trainer_state.json"
@@ -0,0 +1,19283 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.1014,
+  "global_step": 2535,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 2e-06,
+      "loss": 2.3586,
+      "theoretical_loss": 3.484799908140819,
+      "tokens_seen": 1661665280
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4e-06,
+      "loss": 2.2339,
+      "theoretical_loss": 3.4847880270596066,
+      "tokens_seen": 1661730816
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6e-06,
+      "loss": 2.2016,
+      "theoretical_loss": 3.4847761465781497,
+      "tokens_seen": 1661796352
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8e-06,
+      "loss": 2.2196,
+      "theoretical_loss": 3.484764266696394,
+      "tokens_seen": 1661861888
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1e-05,
+      "loss": 2.1452,
+      "theoretical_loss": 3.4847523874142867,
+      "tokens_seen": 1661927424
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.2e-05,
+      "loss": 2.1924,
+      "theoretical_loss": 3.4847405087317727,
+      "tokens_seen": 1661992960
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.4e-05,
+      "loss": 2.3474,
+      "theoretical_loss": 3.4847286306487986,
+      "tokens_seen": 1662058496
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.6e-05,
+      "loss": 2.3439,
+      "theoretical_loss": 3.48471675316531,
+      "tokens_seen": 1662124032
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.8e-05,
+      "loss": 2.1693,
+      "theoretical_loss": 3.4847048762812536,
+      "tokens_seen": 1662189568
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2e-05,
+      "loss": 2.0245,
+      "theoretical_loss": 3.4846929999965757,
+      "tokens_seen": 1662255104
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.2e-05,
+      "loss": 2.071,
+      "theoretical_loss": 3.484681124311222,
+      "tokens_seen": 1662320640
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.4e-05,
+      "loss": 1.9012,
+      "theoretical_loss": 3.4846692492251385,
+      "tokens_seen": 1662386176
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.6e-05,
+      "loss": 1.8901,
+      "theoretical_loss": 3.484657374738272,
+      "tokens_seen": 1662451712
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.8e-05,
+      "loss": 1.9415,
+      "theoretical_loss": 3.4846455008505677,
+      "tokens_seen": 1662517248
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3e-05,
+      "loss": 1.9999,
+      "theoretical_loss": 3.484633627561972,
+      "tokens_seen": 1662582784
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.2e-05,
+      "loss": 1.9898,
+      "theoretical_loss": 3.484621754872432,
+      "tokens_seen": 1662648320
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.4000000000000007e-05,
+      "loss": 1.6089,
+      "theoretical_loss": 3.4846098827818928,
+      "tokens_seen": 1662713856
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.6e-05,
+      "loss": 1.7149,
+      "theoretical_loss": 3.4845980112903003,
+      "tokens_seen": 1662779392
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.8e-05,
+      "loss": 1.6944,
+      "theoretical_loss": 3.484586140397602,
+      "tokens_seen": 1662844928
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4e-05,
+      "loss": 1.4261,
+      "theoretical_loss": 3.484574270103743,
+      "tokens_seen": 1662910464
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.2000000000000004e-05,
+      "loss": 1.7173,
+      "theoretical_loss": 3.484562400408671,
+      "tokens_seen": 1662976000
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.4e-05,
+      "loss": 1.5526,
+      "theoretical_loss": 3.4845505313123297,
+      "tokens_seen": 1663041536
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.6e-05,
+      "loss": 1.7101,
+      "theoretical_loss": 3.4845386628146677,
+      "tokens_seen": 1663107072
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 4.8e-05,
+      "loss": 1.5717,
+      "theoretical_loss": 3.4845267949156296,
+      "tokens_seen": 1663172608
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/aligned_token_count": 0.4392717480659485,
+      "objective/train/avg_token_score": 0.027796979993581772,
+      "objective/train/docs_used": 941634,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.5729044675827026,
+      "objective/train/original_loss": 3.6452524662017822,
+      "objective/train/theoretical_loss": 3.4845149276151623,
+      "objective/train/tokens_used": 22097376,
+      "objective/train/unlikelihood_loss": 0.09146079421043396,
+      "theoretical_loss": 3.4845149276151623,
+      "tokens_seen": 1663238144
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5e-05,
+      "loss": 1.6885,
+      "theoretical_loss": 3.4845149276151623,
+      "tokens_seen": 1663238144
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.2e-05,
+      "loss": 1.4219,
+      "theoretical_loss": 3.484503060913212,
+      "tokens_seen": 1663303680
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.4e-05,
+      "loss": 1.5592,
+      "theoretical_loss": 3.484491194809725,
+      "tokens_seen": 1663369216
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.6e-05,
+      "loss": 1.1071,
+      "theoretical_loss": 3.484479329304647,
+      "tokens_seen": 1663434752
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.800000000000001e-05,
+      "loss": 1.4801,
+      "theoretical_loss": 3.484467464397925,
+      "tokens_seen": 1663500288
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6e-05,
+      "loss": 1.6332,
+      "theoretical_loss": 3.484455600089505,
+      "tokens_seen": 1663565824
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.2e-05,
+      "loss": 1.4538,
+      "theoretical_loss": 3.4844437363793332,
+      "tokens_seen": 1663631360
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.4e-05,
+      "loss": 1.5288,
+      "theoretical_loss": 3.4844318732673556,
+      "tokens_seen": 1663696896
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.6e-05,
+      "loss": 1.4459,
+      "theoretical_loss": 3.484420010753519,
+      "tokens_seen": 1663762432
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.800000000000001e-05,
+      "loss": 1.4075,
+      "theoretical_loss": 3.484408148837769,
+      "tokens_seen": 1663827968
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.000000000000001e-05,
+      "loss": 1.2731,
+      "theoretical_loss": 3.4843962875200525,
+      "tokens_seen": 1663893504
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.2e-05,
+      "loss": 1.2299,
+      "theoretical_loss": 3.484384426800316,
+      "tokens_seen": 1663959040
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.4e-05,
+      "loss": 1.6132,
+      "theoretical_loss": 3.4843725666785046,
+      "tokens_seen": 1664024576
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.6e-05,
+      "loss": 1.639,
+      "theoretical_loss": 3.484360707154566,
+      "tokens_seen": 1664090112
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 7.8e-05,
+      "loss": 1.3998,
+      "theoretical_loss": 3.484348848228446,
+      "tokens_seen": 1664155648
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8e-05,
+      "loss": 1.6466,
+      "theoretical_loss": 3.4843369899000907,
+      "tokens_seen": 1664221184
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.2e-05,
+      "loss": 1.4191,
+      "theoretical_loss": 3.484325132169446,
+      "tokens_seen": 1664286720
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.400000000000001e-05,
+      "loss": 1.3433,
+      "theoretical_loss": 3.4843132750364596,
+      "tokens_seen": 1664352256
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.599999999999999e-05,
+      "loss": 1.6113,
+      "theoretical_loss": 3.4843014185010768,
+      "tokens_seen": 1664417792
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.8e-05,
+      "loss": 1.5214,
+      "theoretical_loss": 3.4842895625632444,
+      "tokens_seen": 1664483328
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 8.999999999999999e-05,
+      "loss": 1.5353,
+      "theoretical_loss": 3.4842777072229083,
+      "tokens_seen": 1664548864
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.2e-05,
+      "loss": 1.4214,
+      "theoretical_loss": 3.484265852480015,
+      "tokens_seen": 1664614400
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.400000000000001e-05,
+      "loss": 1.4677,
+      "theoretical_loss": 3.4842539983345118,
+      "tokens_seen": 1664679936
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.6e-05,
+      "loss": 1.4979,
+      "theoretical_loss": 3.484242144786344,
+      "tokens_seen": 1664745472
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 9.800000000000001e-05,
+      "loss": 1.5655,
+      "theoretical_loss": 3.4842302918354577,
+      "tokens_seen": 1664811008
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/aligned_token_count": 0.4052419364452362,
+      "objective/train/avg_token_score": 0.023708028718829155,
+      "objective/train/docs_used": 943199,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.3439230918884277,
+      "objective/train/original_loss": 3.491624593734741,
+      "objective/train/theoretical_loss": 3.4842184394818,
+      "objective/train/tokens_used": 23735776,
+      "objective/train/unlikelihood_loss": 0.11052291095256805,
+      "theoretical_loss": 3.4842184394818,
+      "tokens_seen": 1664876544
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0001,
+      "loss": 1.2824,
+      "theoretical_loss": 3.4842184394818,
+      "tokens_seen": 1664876544
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000102,
+      "loss": 1.3515,
+      "theoretical_loss": 3.4842065877253177,
+      "tokens_seen": 1664942080
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000104,
+      "loss": 1.327,
+      "theoretical_loss": 3.4841947365659562,
+      "tokens_seen": 1665007616
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000106,
+      "loss": 1.4169,
+      "theoretical_loss": 3.4841828860036625,
+      "tokens_seen": 1665073152
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000108,
+      "loss": 1.4514,
+      "theoretical_loss": 3.484171036038383,
+      "tokens_seen": 1665138688
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011,
+      "loss": 1.374,
+      "theoretical_loss": 3.4841591866700643,
+      "tokens_seen": 1665204224
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000112,
+      "loss": 1.5037,
+      "theoretical_loss": 3.4841473378986523,
+      "tokens_seen": 1665269760
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000114,
+      "loss": 1.5909,
+      "theoretical_loss": 3.4841354897240935,
+      "tokens_seen": 1665335296
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00011600000000000001,
+      "loss": 1.3882,
+      "theoretical_loss": 3.4841236421463346,
+      "tokens_seen": 1665400832
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000118,
+      "loss": 1.4278,
+      "theoretical_loss": 3.4841117951653224,
+      "tokens_seen": 1665466368
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00012,
+      "loss": 1.2305,
+      "theoretical_loss": 3.4840999487810027,
+      "tokens_seen": 1665531904
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000122,
+      "loss": 1.3588,
+      "theoretical_loss": 3.4840881029933226,
+      "tokens_seen": 1665597440
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000124,
+      "loss": 1.1805,
+      "theoretical_loss": 3.484076257802228,
+      "tokens_seen": 1665662976
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000126,
+      "loss": 1.4335,
+      "theoretical_loss": 3.4840644132076655,
+      "tokens_seen": 1665728512
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000128,
+      "loss": 1.5079,
+      "theoretical_loss": 3.484052569209582,
+      "tokens_seen": 1665794048
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013000000000000002,
+      "loss": 1.319,
+      "theoretical_loss": 3.4840407258079233,
+      "tokens_seen": 1665859584
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000132,
+      "loss": 1.4067,
+      "theoretical_loss": 3.4840288830026367,
+      "tokens_seen": 1665925120
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000134,
+      "loss": 1.2983,
+      "theoretical_loss": 3.484017040793668,
+      "tokens_seen": 1665990656
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013600000000000003,
+      "loss": 1.2986,
+      "theoretical_loss": 3.4840051991809644,
+      "tokens_seen": 1666056192
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00013800000000000002,
+      "loss": 1.2998,
+      "theoretical_loss": 3.4839933581644718,
+      "tokens_seen": 1666121728
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014000000000000001,
+      "loss": 1.5612,
+      "theoretical_loss": 3.4839815177441373,
+      "tokens_seen": 1666187264
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00014199999999999998,
+      "loss": 1.3289,
+      "theoretical_loss": 3.4839696779199065,
+      "tokens_seen": 1666252800
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000144,
+      "loss": 1.4071,
+      "theoretical_loss": 3.483957838691727,
+      "tokens_seen": 1666318336
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000146,
+      "loss": 1.1692,
+      "theoretical_loss": 3.483946000059545,
+      "tokens_seen": 1666383872
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000148,
+      "loss": 1.4058,
+      "theoretical_loss": 3.483934162023307,
+      "tokens_seen": 1666449408
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/aligned_token_count": 0.3277125954627991,
+      "objective/train/avg_token_score": 0.01894918829202652,
+      "objective/train/docs_used": 943856,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.0503660440444946,
+      "objective/train/original_loss": 3.4445393085479736,
+      "objective/train/theoretical_loss": 3.4839223245829594,
+      "objective/train/tokens_used": 25374176,
+      "objective/train/unlikelihood_loss": 0.0962013378739357,
+      "theoretical_loss": 3.4839223245829594,
+      "tokens_seen": 1666514944
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00015,
+      "loss": 1.1905,
+      "theoretical_loss": 3.4839223245829594,
+      "tokens_seen": 1666514944
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000152,
+      "loss": 1.5246,
+      "theoretical_loss": 3.4839104877384486,
+      "tokens_seen": 1666580480
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000154,
+      "loss": 1.3905,
+      "theoretical_loss": 3.483898651489722,
+      "tokens_seen": 1666646016
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000156,
+      "loss": 1.4391,
+      "theoretical_loss": 3.4838868158367253,
+      "tokens_seen": 1666711552
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000158,
+      "loss": 1.2598,
+      "theoretical_loss": 3.4838749807794063,
+      "tokens_seen": 1666777088
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016,
+      "loss": 1.4413,
+      "theoretical_loss": 3.4838631463177103,
+      "tokens_seen": 1666842624
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000162,
+      "loss": 1.5131,
+      "theoretical_loss": 3.483851312451584,
+      "tokens_seen": 1666908160
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000164,
+      "loss": 1.333,
+      "theoretical_loss": 3.4838394791809746,
+      "tokens_seen": 1666973696
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016600000000000002,
+      "loss": 1.6326,
+      "theoretical_loss": 3.4838276465058287,
+      "tokens_seen": 1667039232
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00016800000000000002,
+      "loss": 1.5681,
+      "theoretical_loss": 3.4838158144260927,
+      "tokens_seen": 1667104768
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017,
+      "loss": 1.1809,
+      "theoretical_loss": 3.483803982941714,
+      "tokens_seen": 1667170304
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017199999999999998,
+      "loss": 1.2542,
+      "theoretical_loss": 3.4837921520526374,
+      "tokens_seen": 1667235840
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000174,
+      "loss": 1.3605,
+      "theoretical_loss": 3.483780321758811,
+      "tokens_seen": 1667301376
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000176,
+      "loss": 1.6095,
+      "theoretical_loss": 3.4837684920601815,
+      "tokens_seen": 1667366912
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000178,
+      "loss": 1.4505,
+      "theoretical_loss": 3.483756662956695,
+      "tokens_seen": 1667432448
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00017999999999999998,
+      "loss": 1.2638,
+      "theoretical_loss": 3.483744834448298,
+      "tokens_seen": 1667497984
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000182,
+      "loss": 1.4185,
+      "theoretical_loss": 3.483733006534938,
+      "tokens_seen": 1667563520
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000184,
+      "loss": 1.2676,
+      "theoretical_loss": 3.483721179216561,
+      "tokens_seen": 1667629056
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000186,
+      "loss": 1.7296,
+      "theoretical_loss": 3.4837093524931144,
+      "tokens_seen": 1667694592
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00018800000000000002,
+      "loss": 1.6747,
+      "theoretical_loss": 3.4836975263645438,
+      "tokens_seen": 1667760128
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019,
+      "loss": 1.463,
+      "theoretical_loss": 3.483685700830797,
+      "tokens_seen": 1667825664
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000192,
+      "loss": 1.2903,
+      "theoretical_loss": 3.48367387589182,
+      "tokens_seen": 1667891200
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000194,
+      "loss": 1.3391,
+      "theoretical_loss": 3.48366205154756,
+      "tokens_seen": 1667956736
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019600000000000002,
+      "loss": 1.6197,
+      "theoretical_loss": 3.4836502277979626,
+      "tokens_seen": 1668022272
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00019800000000000002,
+      "loss": 1.3482,
+      "theoretical_loss": 3.4836384046429765,
+      "tokens_seen": 1668087808
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/aligned_token_count": 0.3819648027420044,
+      "objective/train/avg_token_score": 0.015858473256230354,
+      "objective/train/docs_used": 945126,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.2602568864822388,
+      "objective/train/original_loss": 3.6047937870025635,
+      "objective/train/theoretical_loss": 3.4836265820825467,
+      "objective/train/tokens_used": 27012576,
+      "objective/train/unlikelihood_loss": 0.07870878279209137,
+      "theoretical_loss": 3.4836265820825467,
+      "tokens_seen": 1668153344
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.0002,
+      "loss": 1.3153,
+      "theoretical_loss": 3.4836265820825467,
+      "tokens_seen": 1668153344
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000202,
+      "loss": 1.313,
+      "theoretical_loss": 3.483614760116621,
+      "tokens_seen": 1668218880
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000204,
+      "loss": 1.4786,
+      "theoretical_loss": 3.4836029387451455,
+      "tokens_seen": 1668284416
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000206,
+      "loss": 1.4841,
+      "theoretical_loss": 3.483591117968067,
+      "tokens_seen": 1668349952
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000208,
+      "loss": 1.4115,
+      "theoretical_loss": 3.483579297785333,
+      "tokens_seen": 1668415488
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00021,
+      "loss": 1.5871,
+      "theoretical_loss": 3.483567478196889,
+      "tokens_seen": 1668481024
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000212,
+      "loss": 1.5155,
+      "theoretical_loss": 3.483555659202683,
+      "tokens_seen": 1668546560
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000214,
+      "loss": 1.6009,
+      "theoretical_loss": 3.4835438408026613,
+      "tokens_seen": 1668612096
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000216,
+      "loss": 1.1647,
+      "theoretical_loss": 3.483532022996771,
+      "tokens_seen": 1668677632
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000218,
+      "loss": 1.3228,
+      "theoretical_loss": 3.483520205784958,
+      "tokens_seen": 1668743168
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022,
+      "loss": 1.4557,
+      "theoretical_loss": 3.48350838916717,
+      "tokens_seen": 1668808704
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000222,
+      "loss": 1.3735,
+      "theoretical_loss": 3.4834965731433534,
+      "tokens_seen": 1668874240
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000224,
+      "loss": 1.3874,
+      "theoretical_loss": 3.4834847577134553,
+      "tokens_seen": 1668939776
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00022600000000000002,
+      "loss": 1.3996,
+      "theoretical_loss": 3.4834729428774223,
+      "tokens_seen": 1669005312
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000228,
+      "loss": 1.2483,
+      "theoretical_loss": 3.4834611286352013,
+      "tokens_seen": 1669070848
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023,
+      "loss": 1.5426,
+      "theoretical_loss": 3.4834493149867387,
+      "tokens_seen": 1669136384
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023200000000000003,
+      "loss": 1.1371,
+      "theoretical_loss": 3.483437501931982,
+      "tokens_seen": 1669201920
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023400000000000002,
+      "loss": 1.5659,
+      "theoretical_loss": 3.483425689470878,
+      "tokens_seen": 1669267456
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000236,
+      "loss": 1.5264,
+      "theoretical_loss": 3.4834138776033736,
+      "tokens_seen": 1669332992
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00023799999999999998,
+      "loss": 1.3228,
+      "theoretical_loss": 3.483402066329415,
+      "tokens_seen": 1669398528
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.00024,
+      "loss": 1.4733,
+      "theoretical_loss": 3.48339025564895,
+      "tokens_seen": 1669464064
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000242,
+      "loss": 1.329,
+      "theoretical_loss": 3.4833784455619248,
+      "tokens_seen": 1669529600
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000244,
+      "loss": 1.4319,
+      "theoretical_loss": 3.4833666360682867,
+      "tokens_seen": 1669595136
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000246,
+      "loss": 1.4501,
+      "theoretical_loss": 3.4833548271679824,
+      "tokens_seen": 1669660672
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 0.000248,
+      "loss": 1.3319,
+      "theoretical_loss": 3.4833430188609587,
+      "tokens_seen": 1669726208
+    },
+    {
+      "epoch": 0.0,
+      "objective/train/aligned_token_count": 0.45992180705070496,
+      "objective/train/avg_token_score": 0.009118808433413506,
+      "objective/train/docs_used": 945699,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.6401410102844238,
+      "objective/train/original_loss": 4.021496772766113,
+      "objective/train/theoretical_loss": 3.4833312111471626,
+      "objective/train/tokens_used": 28650976,
+      "objective/train/unlikelihood_loss": 0.06683876365423203,
+      "theoretical_loss": 3.4833312111471626,
+      "tokens_seen": 1669791744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025,
+      "loss": 1.6621,
+      "theoretical_loss": 3.4833312111471626,
+      "tokens_seen": 1669791744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000252,
+      "loss": 1.361,
+      "theoretical_loss": 3.483319404026541,
+      "tokens_seen": 1669857280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000254,
+      "loss": 1.1887,
+      "theoretical_loss": 3.4833075974990404,
+      "tokens_seen": 1669922816
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000256,
+      "loss": 1.4238,
+      "theoretical_loss": 3.4832957915646086,
+      "tokens_seen": 1669988352
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00025800000000000004,
+      "loss": 1.4582,
+      "theoretical_loss": 3.4832839862231926,
+      "tokens_seen": 1670053888
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00026000000000000003,
+      "loss": 1.509,
+      "theoretical_loss": 3.4832721814747383,
+      "tokens_seen": 1670119424
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000262,
+      "loss": 1.6246,
+      "theoretical_loss": 3.4832603773191932,
+      "tokens_seen": 1670184960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000264,
+      "loss": 1.3161,
+      "theoretical_loss": 3.4832485737565047,
+      "tokens_seen": 1670250496
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000266,
+      "loss": 1.6053,
+      "theoretical_loss": 3.483236770786619,
+      "tokens_seen": 1670316032
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000268,
+      "loss": 1.2865,
+      "theoretical_loss": 3.4832249684094836,
+      "tokens_seen": 1670381568
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00027,
+      "loss": 1.3504,
+      "theoretical_loss": 3.4832131666250454,
+      "tokens_seen": 1670447104
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00027200000000000005,
+      "loss": 1.4324,
+      "theoretical_loss": 3.483201365433251,
+      "tokens_seen": 1670512640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00027400000000000005,
+      "loss": 1.478,
+      "theoretical_loss": 3.4831895648340474,
+      "tokens_seen": 1670578176
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00027600000000000004,
+      "loss": 1.3201,
+      "theoretical_loss": 3.4831777648273823,
+      "tokens_seen": 1670643712
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00027800000000000004,
+      "loss": 1.3004,
+      "theoretical_loss": 3.4831659654132023,
+      "tokens_seen": 1670709248
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00028000000000000003,
+      "loss": 1.2886,
+      "theoretical_loss": 3.4831541665914543,
+      "tokens_seen": 1670774784
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00028199999999999997,
+      "loss": 1.4002,
+      "theoretical_loss": 3.4831423683620852,
+      "tokens_seen": 1670840320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00028399999999999996,
+      "loss": 1.0925,
+      "theoretical_loss": 3.4831305707250424,
+      "tokens_seen": 1670905856
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00028599999999999996,
+      "loss": 1.594,
+      "theoretical_loss": 3.483118773680273,
+      "tokens_seen": 1670971392
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000288,
+      "loss": 1.5114,
+      "theoretical_loss": 3.4831069772277234,
+      "tokens_seen": 1671036928
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00029,
+      "loss": 1.4329,
+      "theoretical_loss": 3.483095181367341,
+      "tokens_seen": 1671102464
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000292,
+      "loss": 1.5268,
+      "theoretical_loss": 3.4830833860990733,
+      "tokens_seen": 1671168000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000294,
+      "loss": 1.3822,
+      "theoretical_loss": 3.4830715914228665,
+      "tokens_seen": 1671233536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000296,
+      "loss": 1.5873,
+      "theoretical_loss": 3.4830597973386683,
+      "tokens_seen": 1671299072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000298,
+      "loss": 1.5036,
+      "theoretical_loss": 3.4830480038464255,
+      "tokens_seen": 1671364608
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/aligned_token_count": 0.3960166275501251,
+      "objective/train/avg_token_score": 0.020477375015616417,
+      "objective/train/docs_used": 946982,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.2867282629013062,
+      "objective/train/original_loss": 4.248522758483887,
+      "objective/train/theoretical_loss": 3.4830362109460857,
+      "objective/train/tokens_used": 30289376,
+      "objective/train/unlikelihood_loss": 0.06127317622303963,
+      "theoretical_loss": 3.4830362109460857,
+      "tokens_seen": 1671430144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0003,
+      "loss": 1.2874,
+      "theoretical_loss": 3.4830362109460857,
+      "tokens_seen": 1671430144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000302,
+      "loss": 1.5552,
+      "theoretical_loss": 3.483024418637595,
+      "tokens_seen": 1671495680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000304,
+      "loss": 1.4532,
+      "theoretical_loss": 3.4830126269209014,
+      "tokens_seen": 1671561216
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000306,
+      "loss": 1.3209,
+      "theoretical_loss": 3.4830008357959517,
+      "tokens_seen": 1671626752
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000308,
+      "loss": 1.5412,
+      "theoretical_loss": 3.4829890452626926,
+      "tokens_seen": 1671692288
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00031,
+      "loss": 1.0823,
+      "theoretical_loss": 3.482977255321072,
+      "tokens_seen": 1671757824
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000312,
+      "loss": 1.4433,
+      "theoretical_loss": 3.482965465971036,
+      "tokens_seen": 1671823360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000314,
+      "loss": 1.3343,
+      "theoretical_loss": 3.4829536772125325,
+      "tokens_seen": 1671888896
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000316,
+      "loss": 1.4329,
+      "theoretical_loss": 3.482941889045509,
+      "tokens_seen": 1671954432
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00031800000000000003,
+      "loss": 1.2974,
+      "theoretical_loss": 3.4829301014699117,
+      "tokens_seen": 1672019968
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00032,
+      "loss": 1.5543,
+      "theoretical_loss": 3.482918314485688,
+      "tokens_seen": 1672085504
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000322,
+      "loss": 1.6948,
+      "theoretical_loss": 3.4829065280927853,
+      "tokens_seen": 1672151040
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000324,
+      "loss": 1.358,
+      "theoretical_loss": 3.482894742291151,
+      "tokens_seen": 1672216576
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000326,
+      "loss": 1.5256,
+      "theoretical_loss": 3.4828829570807316,
+      "tokens_seen": 1672282112
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000328,
+      "loss": 1.2557,
+      "theoretical_loss": 3.4828711724614747,
+      "tokens_seen": 1672347648
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00033,
+      "loss": 1.4626,
+      "theoretical_loss": 3.4828593884333277,
+      "tokens_seen": 1672413184
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00033200000000000005,
+      "loss": 1.4653,
+      "theoretical_loss": 3.4828476049962367,
+      "tokens_seen": 1672478720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00033400000000000004,
+      "loss": 1.4492,
+      "theoretical_loss": 3.4828358221501503,
+      "tokens_seen": 1672544256
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00033600000000000004,
+      "loss": 1.3397,
+      "theoretical_loss": 3.4828240398950143,
+      "tokens_seen": 1672609792
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00033800000000000003,
+      "loss": 1.2303,
+      "theoretical_loss": 3.4828122582307777,
+      "tokens_seen": 1672675328
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00034,
+      "loss": 1.3492,
+      "theoretical_loss": 3.482800477157386,
+      "tokens_seen": 1672740864
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000342,
+      "loss": 1.4296,
+      "theoretical_loss": 3.4827886966747874,
+      "tokens_seen": 1672806400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00034399999999999996,
+      "loss": 1.6639,
+      "theoretical_loss": 3.482776916782929,
+      "tokens_seen": 1672871936
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000346,
+      "loss": 1.3111,
+      "theoretical_loss": 3.482765137481757,
+      "tokens_seen": 1672937472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000348,
+      "loss": 1.4424,
+      "theoretical_loss": 3.4827533587712205,
+      "tokens_seen": 1673003008
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/aligned_token_count": 0.42277616262435913,
+      "objective/train/avg_token_score": 0.034132879227399826,
+      "objective/train/docs_used": 947668,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.4029995203018188,
+      "objective/train/original_loss": 4.207383155822754,
+      "objective/train/theoretical_loss": 3.4827415806512656,
+      "objective/train/tokens_used": 31927776,
+      "objective/train/unlikelihood_loss": 0.05666952580213547,
+      "theoretical_loss": 3.4827415806512656,
+      "tokens_seen": 1673068544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00035,
+      "loss": 1.7249,
+      "theoretical_loss": 3.4827415806512656,
+      "tokens_seen": 1673068544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000352,
+      "loss": 1.4411,
+      "theoretical_loss": 3.4827298031218397,
+      "tokens_seen": 1673134080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000354,
+      "loss": 1.4599,
+      "theoretical_loss": 3.48271802618289,
+      "tokens_seen": 1673199616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000356,
+      "loss": 1.4334,
+      "theoretical_loss": 3.4827062498343637,
+      "tokens_seen": 1673265152
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000358,
+      "loss": 1.4262,
+      "theoretical_loss": 3.4826944740762085,
+      "tokens_seen": 1673330688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00035999999999999997,
+      "loss": 1.5422,
+      "theoretical_loss": 3.4826826989083712,
+      "tokens_seen": 1673396224
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000362,
+      "loss": 1.3631,
+      "theoretical_loss": 3.4826709243307996,
+      "tokens_seen": 1673461760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000364,
+      "loss": 1.5299,
+      "theoretical_loss": 3.4826591503434408,
+      "tokens_seen": 1673527296
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000366,
+      "loss": 1.4358,
+      "theoretical_loss": 3.4826473769462414,
+      "tokens_seen": 1673592832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000368,
+      "loss": 1.3031,
+      "theoretical_loss": 3.4826356041391495,
+      "tokens_seen": 1673658368
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00037,
+      "loss": 1.372,
+      "theoretical_loss": 3.482623831922113,
+      "tokens_seen": 1673723904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000372,
+      "loss": 1.4469,
+      "theoretical_loss": 3.4826120602950774,
+      "tokens_seen": 1673789440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000374,
+      "loss": 1.3485,
+      "theoretical_loss": 3.4826002892579915,
+      "tokens_seen": 1673854976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00037600000000000003,
+      "loss": 1.1995,
+      "theoretical_loss": 3.482588518810802,
+      "tokens_seen": 1673920512
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000378,
+      "loss": 1.4059,
+      "theoretical_loss": 3.4825767489534574,
+      "tokens_seen": 1673986048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00038,
+      "loss": 1.4645,
+      "theoretical_loss": 3.482564979685903,
+      "tokens_seen": 1674051584
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000382,
+      "loss": 1.6177,
+      "theoretical_loss": 3.482553211008088,
+      "tokens_seen": 1674117120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000384,
+      "loss": 1.5993,
+      "theoretical_loss": 3.482541442919959,
+      "tokens_seen": 1674182656
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000386,
+      "loss": 1.5792,
+      "theoretical_loss": 3.482529675421463,
+      "tokens_seen": 1674248192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000388,
+      "loss": 1.2894,
+      "theoretical_loss": 3.4825179085125484,
+      "tokens_seen": 1674313728
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00039000000000000005,
+      "loss": 1.3151,
+      "theoretical_loss": 3.4825061421931616,
+      "tokens_seen": 1674379264
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00039200000000000004,
+      "loss": 1.484,
+      "theoretical_loss": 3.4824943764632503,
+      "tokens_seen": 1674444800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00039400000000000004,
+      "loss": 1.4304,
+      "theoretical_loss": 3.4824826113227623,
+      "tokens_seen": 1674510336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00039600000000000003,
+      "loss": 1.3797,
+      "theoretical_loss": 3.4824708467716445,
+      "tokens_seen": 1674575872
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000398,
+      "loss": 1.1721,
+      "theoretical_loss": 3.4824590828098447,
+      "tokens_seen": 1674641408
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/aligned_token_count": 0.47403469681739807,
+      "objective/train/avg_token_score": 0.020986413583159447,
+      "objective/train/docs_used": 948408,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.4957624673843384,
+      "objective/train/original_loss": 4.087368488311768,
+      "objective/train/theoretical_loss": 3.48244731943731,
+      "objective/train/tokens_used": 33566176,
+      "objective/train/unlikelihood_loss": 0.058626897633075714,
+      "theoretical_loss": 3.48244731943731,
+      "tokens_seen": 1674706944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004,
+      "loss": 1.6842,
+      "theoretical_loss": 3.48244731943731,
+      "tokens_seen": 1674706944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000402,
+      "loss": 1.3746,
+      "theoretical_loss": 3.4824355566539875,
+      "tokens_seen": 1674772480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000404,
+      "loss": 1.4025,
+      "theoretical_loss": 3.4824237944598257,
+      "tokens_seen": 1674838016
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00040600000000000006,
+      "loss": 1.4322,
+      "theoretical_loss": 3.482412032854771,
+      "tokens_seen": 1674903552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000408,
+      "loss": 1.5092,
+      "theoretical_loss": 3.482400271838771,
+      "tokens_seen": 1674969088
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00041,
+      "loss": 1.3048,
+      "theoretical_loss": 3.482388511411774,
+      "tokens_seen": 1675034624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000412,
+      "loss": 1.1327,
+      "theoretical_loss": 3.4823767515737267,
+      "tokens_seen": 1675100160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000414,
+      "loss": 1.3768,
+      "theoretical_loss": 3.4823649923245767,
+      "tokens_seen": 1675165696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000416,
+      "loss": 1.3944,
+      "theoretical_loss": 3.4823532336642717,
+      "tokens_seen": 1675231232
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00041799999999999997,
+      "loss": 1.5612,
+      "theoretical_loss": 3.482341475592759,
+      "tokens_seen": 1675296768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00042,
+      "loss": 1.6033,
+      "theoretical_loss": 3.4823297181099857,
+      "tokens_seen": 1675362304
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000422,
+      "loss": 1.2907,
+      "theoretical_loss": 3.4823179612159,
+      "tokens_seen": 1675427840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000424,
+      "loss": 1.4496,
+      "theoretical_loss": 3.4823062049104485,
+      "tokens_seen": 1675493376
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000426,
+      "loss": 1.3219,
+      "theoretical_loss": 3.4822944491935797,
+      "tokens_seen": 1675558912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000428,
+      "loss": 1.6165,
+      "theoretical_loss": 3.4822826940652405,
+      "tokens_seen": 1675624448
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00043,
+      "loss": 1.46,
+      "theoretical_loss": 3.482270939525379,
+      "tokens_seen": 1675689984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000432,
+      "loss": 1.3409,
+      "theoretical_loss": 3.482259185573942,
+      "tokens_seen": 1675755520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00043400000000000003,
+      "loss": 1.5918,
+      "theoretical_loss": 3.4822474322108774,
+      "tokens_seen": 1675821056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000436,
+      "loss": 1.3411,
+      "theoretical_loss": 3.4822356794361324,
+      "tokens_seen": 1675886592
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000438,
+      "loss": 1.4671,
+      "theoretical_loss": 3.482223927249655,
+      "tokens_seen": 1675952128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00044,
+      "loss": 1.407,
+      "theoretical_loss": 3.482212175651393,
+      "tokens_seen": 1676017664
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000442,
+      "loss": 1.4158,
+      "theoretical_loss": 3.4822004246412934,
+      "tokens_seen": 1676083200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000444,
+      "loss": 1.2442,
+      "theoretical_loss": 3.4821886742193042,
+      "tokens_seen": 1676148736
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000446,
+      "loss": 1.6039,
+      "theoretical_loss": 3.4821769243853717,
+      "tokens_seen": 1676214272
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000448,
+      "loss": 1.3757,
+      "theoretical_loss": 3.4821651751394453,
+      "tokens_seen": 1676279808
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/aligned_token_count": 0.43438416719436646,
+      "objective/train/avg_token_score": 0.02250559814274311,
+      "objective/train/docs_used": 949481,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.3553886413574219,
+      "objective/train/original_loss": 4.167667388916016,
+      "objective/train/theoretical_loss": 3.4821534264814717,
+      "objective/train/tokens_used": 35204576,
+      "objective/train/unlikelihood_loss": 0.05399244651198387,
+      "theoretical_loss": 3.4821534264814717,
+      "tokens_seen": 1676345344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00045000000000000004,
+      "loss": 1.5774,
+      "theoretical_loss": 3.4821534264814717,
+      "tokens_seen": 1676345344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00045200000000000004,
+      "loss": 1.1835,
+      "theoretical_loss": 3.4821416784113985,
+      "tokens_seen": 1676410880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00045400000000000003,
+      "loss": 1.4729,
+      "theoretical_loss": 3.4821299309291733,
+      "tokens_seen": 1676476416
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000456,
+      "loss": 1.4368,
+      "theoretical_loss": 3.4821181840347437,
+      "tokens_seen": 1676541952
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000458,
+      "loss": 1.5042,
+      "theoretical_loss": 3.4821064377280577,
+      "tokens_seen": 1676607488
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046,
+      "loss": 1.4255,
+      "theoretical_loss": 3.4820946920090625,
+      "tokens_seen": 1676673024
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000462,
+      "loss": 1.3812,
+      "theoretical_loss": 3.4820829468777057,
+      "tokens_seen": 1676738560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046400000000000006,
+      "loss": 1.384,
+      "theoretical_loss": 3.482071202333935,
+      "tokens_seen": 1676804096
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046600000000000005,
+      "loss": 1.3564,
+      "theoretical_loss": 3.482059458377698,
+      "tokens_seen": 1676869632
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00046800000000000005,
+      "loss": 1.4955,
+      "theoretical_loss": 3.4820477150089433,
+      "tokens_seen": 1676935168
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00047,
+      "loss": 1.3263,
+      "theoretical_loss": 3.4820359722276164,
+      "tokens_seen": 1677000704
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000472,
+      "loss": 1.6779,
+      "theoretical_loss": 3.4820242300336677,
+      "tokens_seen": 1677066240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000474,
+      "loss": 1.4524,
+      "theoretical_loss": 3.4820124884270425,
+      "tokens_seen": 1677131776
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00047599999999999997,
+      "loss": 1.5956,
+      "theoretical_loss": 3.4820007474076893,
+      "tokens_seen": 1677197312
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00047799999999999996,
+      "loss": 1.5913,
+      "theoretical_loss": 3.481989006975556,
+      "tokens_seen": 1677262848
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00048,
+      "loss": 1.4133,
+      "theoretical_loss": 3.4819772671305906,
+      "tokens_seen": 1677328384
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000482,
+      "loss": 1.2181,
+      "theoretical_loss": 3.48196552787274,
+      "tokens_seen": 1677393920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000484,
+      "loss": 1.337,
+      "theoretical_loss": 3.4819537892019525,
+      "tokens_seen": 1677459456
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000486,
+      "loss": 1.2865,
+      "theoretical_loss": 3.4819420511181756,
+      "tokens_seen": 1677524992
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000488,
+      "loss": 1.6925,
+      "theoretical_loss": 3.481930313621357,
+      "tokens_seen": 1677590528
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00049,
+      "loss": 1.2952,
+      "theoretical_loss": 3.4819185767114442,
+      "tokens_seen": 1677656064
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000492,
+      "loss": 1.3687,
+      "theoretical_loss": 3.4819068403883855,
+      "tokens_seen": 1677721600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000494,
+      "loss": 1.4566,
+      "theoretical_loss": 3.4818951046521276,
+      "tokens_seen": 1677787136
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000496,
+      "loss": 1.3783,
+      "theoretical_loss": 3.4818833695026195,
+      "tokens_seen": 1677852672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498,
+      "loss": 1.4967,
+      "theoretical_loss": 3.4818716349398082,
+      "tokens_seen": 1677918208
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/aligned_token_count": 0.42913001775741577,
+      "objective/train/avg_token_score": 0.028014685958623886,
+      "objective/train/docs_used": 950694,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.2823351621627808,
+      "objective/train/original_loss": 4.018346309661865,
+      "objective/train/theoretical_loss": 3.481859900963642,
+      "objective/train/tokens_used": 36842976,
+      "objective/train/unlikelihood_loss": 0.06534267216920853,
+      "theoretical_loss": 3.481859900963642,
+      "tokens_seen": 1677983744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0005,
+      "loss": 1.4589,
+      "theoretical_loss": 3.481859900963642,
+      "tokens_seen": 1677983744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499979797979798,
+      "loss": 1.7066,
+      "theoretical_loss": 3.481848167574068,
+      "tokens_seen": 1678049280
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999595959595959,
+      "loss": 1.4743,
+      "theoretical_loss": 3.481836434771034,
+      "tokens_seen": 1678114816
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004999393939393939,
+      "loss": 1.4814,
+      "theoretical_loss": 3.4818247025544884,
+      "tokens_seen": 1678180352
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499919191919192,
+      "loss": 1.5105,
+      "theoretical_loss": 3.481812970924379,
+      "tokens_seen": 1678245888
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.00049989898989899,
+      "loss": 1.3163,
+      "theoretical_loss": 3.481801239880652,
+      "tokens_seen": 1678311424
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998787878787879,
+      "loss": 1.5049,
+      "theoretical_loss": 3.4817895094232574,
+      "tokens_seen": 1678376960
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998585858585859,
+      "loss": 1.7115,
+      "theoretical_loss": 3.481777779552141,
+      "tokens_seen": 1678442496
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998383838383839,
+      "loss": 1.3624,
+      "theoretical_loss": 3.481766050267253,
+      "tokens_seen": 1678508032
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004998181818181818,
+      "loss": 1.4109,
+      "theoretical_loss": 3.481754321568539,
+      "tokens_seen": 1678573568
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997979797979798,
+      "loss": 1.6151,
+      "theoretical_loss": 3.4817425934559476,
+      "tokens_seen": 1678639104
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997777777777778,
+      "loss": 1.4839,
+      "theoretical_loss": 3.4817308659294266,
+      "tokens_seen": 1678704640
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997575757575757,
+      "loss": 1.3913,
+      "theoretical_loss": 3.4817191389889244,
+      "tokens_seen": 1678770176
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997373737373737,
+      "loss": 1.4744,
+      "theoretical_loss": 3.481707412634388,
+      "tokens_seen": 1678835712
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004997171717171717,
+      "loss": 1.2652,
+      "theoretical_loss": 3.481695686865766,
+      "tokens_seen": 1678901248
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996969696969697,
+      "loss": 1.5878,
+      "theoretical_loss": 3.481683961683005,
+      "tokens_seen": 1678966784
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996767676767676,
+      "loss": 1.2625,
+      "theoretical_loss": 3.4816722370860544,
+      "tokens_seen": 1679032320
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996565656565657,
+      "loss": 1.3929,
+      "theoretical_loss": 3.4816605130748615,
+      "tokens_seen": 1679097856
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996363636363637,
+      "loss": 1.4169,
+      "theoretical_loss": 3.4816487896493733,
+      "tokens_seen": 1679163392
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004996161616161617,
+      "loss": 1.4403,
+      "theoretical_loss": 3.4816370668095393,
+      "tokens_seen": 1679228928
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995959595959596,
+      "loss": 1.5393,
+      "theoretical_loss": 3.4816253445553063,
+      "tokens_seen": 1679294464
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995757575757576,
+      "loss": 1.6212,
+      "theoretical_loss": 3.4816136228866217,
+      "tokens_seen": 1679360000
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995555555555555,
+      "loss": 1.4312,
+      "theoretical_loss": 3.481601901803435,
+      "tokens_seen": 1679425536
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995353535353536,
+      "loss": 1.4936,
+      "theoretical_loss": 3.481590181305693,
+      "tokens_seen": 1679491072
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004995151515151515,
+      "loss": 1.29,
+      "theoretical_loss": 3.4815784613933434,
+      "tokens_seen": 1679556608
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/aligned_token_count": 0.3876466155052185,
+      "objective/train/avg_token_score": 0.019771317020058632,
+      "objective/train/docs_used": 951339,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.2079968452453613,
+      "objective/train/original_loss": 3.654224395751953,
+      "objective/train/theoretical_loss": 3.481566742066335,
+      "objective/train/tokens_used": 38481376,
+      "objective/train/unlikelihood_loss": 0.10271399468183517,
+      "theoretical_loss": 3.481566742066335,
+      "tokens_seen": 1679622144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994949494949495,
+      "loss": 1.1997,
+      "theoretical_loss": 3.481566742066335,
+      "tokens_seen": 1679622144
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994747474747474,
+      "loss": 1.3423,
+      "theoretical_loss": 3.4815550233246153,
+      "tokens_seen": 1679687680
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994545454545454,
+      "loss": 1.416,
+      "theoretical_loss": 3.481543305168132,
+      "tokens_seen": 1679753216
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994343434343435,
+      "loss": 1.4918,
+      "theoretical_loss": 3.4815315875968333,
+      "tokens_seen": 1679818752
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004994141414141415,
+      "loss": 1.6278,
+      "theoretical_loss": 3.4815198706106676,
+      "tokens_seen": 1679884288
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993939393939394,
+      "loss": 1.4463,
+      "theoretical_loss": 3.481508154209582,
+      "tokens_seen": 1679949824
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993737373737374,
+      "loss": 1.4316,
+      "theoretical_loss": 3.4814964383935245,
+      "tokens_seen": 1680015360
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993535353535354,
+      "loss": 1.5408,
+      "theoretical_loss": 3.4814847231624437,
+      "tokens_seen": 1680080896
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993333333333334,
+      "loss": 1.4201,
+      "theoretical_loss": 3.4814730085162875,
+      "tokens_seen": 1680146432
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004993131313131313,
+      "loss": 1.3325,
+      "theoretical_loss": 3.4814612944550034,
+      "tokens_seen": 1680211968
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992929292929293,
+      "loss": 1.4038,
+      "theoretical_loss": 3.4814495809785395,
+      "tokens_seen": 1680277504
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992727272727272,
+      "loss": 1.4769,
+      "theoretical_loss": 3.4814378680868447,
+      "tokens_seen": 1680343040
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992525252525252,
+      "loss": 1.5646,
+      "theoretical_loss": 3.4814261557798654,
+      "tokens_seen": 1680408576
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992323232323232,
+      "loss": 1.3709,
+      "theoretical_loss": 3.4814144440575507,
+      "tokens_seen": 1680474112
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004992121212121212,
+      "loss": 1.3969,
+      "theoretical_loss": 3.4814027329198485,
+      "tokens_seen": 1680539648
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991919191919192,
+      "loss": 1.4675,
+      "theoretical_loss": 3.481391022366707,
+      "tokens_seen": 1680605184
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991717171717172,
+      "loss": 1.5211,
+      "theoretical_loss": 3.481379312398073,
+      "tokens_seen": 1680670720
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991515151515152,
+      "loss": 1.424,
+      "theoretical_loss": 3.4813676030138963,
+      "tokens_seen": 1680736256
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991313131313132,
+      "loss": 1.285,
+      "theoretical_loss": 3.481355894214124,
+      "tokens_seen": 1680801792
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004991111111111111,
+      "loss": 1.5255,
+      "theoretical_loss": 3.481344185998704,
+      "tokens_seen": 1680867328
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990909090909091,
+      "loss": 1.3467,
+      "theoretical_loss": 3.4813324783675847,
+      "tokens_seen": 1680932864
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990707070707071,
+      "loss": 1.414,
+      "theoretical_loss": 3.481320771320714,
+      "tokens_seen": 1680998400
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004990505050505051,
+      "loss": 1.4323,
+      "theoretical_loss": 3.4813090648580403,
+      "tokens_seen": 1681063936
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499030303030303,
+      "loss": 1.2216,
+      "theoretical_loss": 3.4812973589795106,
+      "tokens_seen": 1681129472
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000499010101010101,
+      "loss": 1.6606,
+      "theoretical_loss": 3.4812856536850747,
+      "tokens_seen": 1681195008
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/aligned_token_count": 0.57746821641922,
+      "objective/train/avg_token_score": 0.003087435383349657,
+      "objective/train/docs_used": 952727,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.965943694114685,
+      "objective/train/original_loss": 3.664557933807373,
+      "objective/train/theoretical_loss": 3.4812739489746796,
+      "objective/train/tokens_used": 40119776,
+      "objective/train/unlikelihood_loss": 0.07379762828350067,
+      "theoretical_loss": 3.4812739489746796,
+      "tokens_seen": 1681260544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498989898989899,
+      "loss": 1.3937,
+      "theoretical_loss": 3.4812739489746796,
+      "tokens_seen": 1681260544
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498969696969697,
+      "loss": 1.4396,
+      "theoretical_loss": 3.4812622448482733,
+      "tokens_seen": 1681326080
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498949494949495,
+      "loss": 1.5074,
+      "theoretical_loss": 3.481250541305804,
+      "tokens_seen": 1681391616
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498929292929293,
+      "loss": 1.3075,
+      "theoretical_loss": 3.4812388383472204,
+      "tokens_seen": 1681457152
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004989090909090909,
+      "loss": 1.4787,
+      "theoretical_loss": 3.4812271359724702,
+      "tokens_seen": 1681522688
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988888888888889,
+      "loss": 1.4177,
+      "theoretical_loss": 3.481215434181501,
+      "tokens_seen": 1681588224
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988686868686869,
+      "loss": 1.3858,
+      "theoretical_loss": 3.481203732974262,
+      "tokens_seen": 1681653760
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988484848484849,
+      "loss": 1.5262,
+      "theoretical_loss": 3.481192032350701,
+      "tokens_seen": 1681719296
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988282828282828,
+      "loss": 1.5809,
+      "theoretical_loss": 3.481180332310765,
+      "tokens_seen": 1681784832
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004988080808080808,
+      "loss": 1.505,
+      "theoretical_loss": 3.481168632854404,
+      "tokens_seen": 1681850368
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987878787878788,
+      "loss": 1.3779,
+      "theoretical_loss": 3.4811569339815644,
+      "tokens_seen": 1681915904
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987676767676767,
+      "loss": 1.3716,
+      "theoretical_loss": 3.4811452356921953,
+      "tokens_seen": 1681981440
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987474747474747,
+      "loss": 1.572,
+      "theoretical_loss": 3.4811335379862456,
+      "tokens_seen": 1682046976
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987272727272728,
+      "loss": 1.4112,
+      "theoretical_loss": 3.4811218408636617,
+      "tokens_seen": 1682112512
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004987070707070708,
+      "loss": 1.4665,
+      "theoretical_loss": 3.481110144324393,
+      "tokens_seen": 1682178048
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986868686868687,
+      "loss": 1.4609,
+      "theoretical_loss": 3.481098448368387,
+      "tokens_seen": 1682243584
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986666666666667,
+      "loss": 1.6894,
+      "theoretical_loss": 3.4810867529955933,
+      "tokens_seen": 1682309120
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986464646464647,
+      "loss": 1.3648,
+      "theoretical_loss": 3.481075058205958,
+      "tokens_seen": 1682374656
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986262626262627,
+      "loss": 1.5816,
+      "theoretical_loss": 3.4810633639994313,
+      "tokens_seen": 1682440192
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004986060606060606,
+      "loss": 1.6544,
+      "theoretical_loss": 3.48105167037596,
+      "tokens_seen": 1682505728
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985858585858586,
+      "loss": 1.4945,
+      "theoretical_loss": 3.4810399773354925,
+      "tokens_seen": 1682571264
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985656565656565,
+      "loss": 1.3454,
+      "theoretical_loss": 3.4810282848779783,
+      "tokens_seen": 1682636800
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985454545454545,
+      "loss": 1.2652,
+      "theoretical_loss": 3.481016593003364,
+      "tokens_seen": 1682702336
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985252525252525,
+      "loss": 1.3969,
+      "theoretical_loss": 3.4810049017115983,
+      "tokens_seen": 1682767872
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004985050505050506,
+      "loss": 1.5008,
+      "theoretical_loss": 3.4809932110026303,
+      "tokens_seen": 1682833408
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/aligned_token_count": 0.4591275751590729,
+      "objective/train/avg_token_score": 0.02467002533376217,
+      "objective/train/docs_used": 953335,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.5312504768371582,
+      "objective/train/original_loss": 4.14010763168335,
+      "objective/train/theoretical_loss": 3.4809815208764077,
+      "objective/train/tokens_used": 41758176,
+      "objective/train/unlikelihood_loss": 0.059290263801813126,
+      "theoretical_loss": 3.4809815208764077,
+      "tokens_seen": 1682898944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984848484848485,
+      "loss": 1.423,
+      "theoretical_loss": 3.4809815208764077,
+      "tokens_seen": 1682898944
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984646464646465,
+      "loss": 1.6025,
+      "theoretical_loss": 3.480969831332878,
+      "tokens_seen": 1682964480
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984444444444445,
+      "loss": 1.6665,
+      "theoretical_loss": 3.4809581423719904,
+      "tokens_seen": 1683030016
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984242424242425,
+      "loss": 1.472,
+      "theoretical_loss": 3.480946453993693,
+      "tokens_seen": 1683095552
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004984040404040404,
+      "loss": 1.4715,
+      "theoretical_loss": 3.480934766197934,
+      "tokens_seen": 1683161088
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983838383838384,
+      "loss": 1.4211,
+      "theoretical_loss": 3.480923078984662,
+      "tokens_seen": 1683226624
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983636363636363,
+      "loss": 1.495,
+      "theoretical_loss": 3.480911392353825,
+      "tokens_seen": 1683292160
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983434343434344,
+      "loss": 1.2509,
+      "theoretical_loss": 3.4808997063053706,
+      "tokens_seen": 1683357696
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983232323232323,
+      "loss": 1.3817,
+      "theoretical_loss": 3.480888020839248,
+      "tokens_seen": 1683423232
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004983030303030303,
+      "loss": 1.2817,
+      "theoretical_loss": 3.480876335955406,
+      "tokens_seen": 1683488768
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982828282828282,
+      "loss": 1.4437,
+      "theoretical_loss": 3.4808646516537918,
+      "tokens_seen": 1683554304
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982626262626263,
+      "loss": 1.5573,
+      "theoretical_loss": 3.4808529679343536,
+      "tokens_seen": 1683619840
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982424242424243,
+      "loss": 1.6952,
+      "theoretical_loss": 3.480841284797041,
+      "tokens_seen": 1683685376
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982222222222223,
+      "loss": 1.2028,
+      "theoretical_loss": 3.4808296022418013,
+      "tokens_seen": 1683750912
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004982020202020202,
+      "loss": 1.3336,
+      "theoretical_loss": 3.480817920268583,
+      "tokens_seen": 1683816448
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981818181818182,
+      "loss": 1.3224,
+      "theoretical_loss": 3.480806238877335,
+      "tokens_seen": 1683881984
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981616161616162,
+      "loss": 1.392,
+      "theoretical_loss": 3.4807945580680055,
+      "tokens_seen": 1683947520
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981414141414142,
+      "loss": 1.3183,
+      "theoretical_loss": 3.480782877840542,
+      "tokens_seen": 1684013056
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981212121212121,
+      "loss": 1.5275,
+      "theoretical_loss": 3.480771198194894,
+      "tokens_seen": 1684078592
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004981010101010101,
+      "loss": 1.3843,
+      "theoretical_loss": 3.4807595191310092,
+      "tokens_seen": 1684144128
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498080808080808,
+      "loss": 1.4248,
+      "theoretical_loss": 3.4807478406488364,
+      "tokens_seen": 1684209664
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498060606060606,
+      "loss": 1.5212,
+      "theoretical_loss": 3.4807361627483235,
+      "tokens_seen": 1684275200
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498040404040404,
+      "loss": 1.5897,
+      "theoretical_loss": 3.4807244854294197,
+      "tokens_seen": 1684340736
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004980202020202021,
+      "loss": 1.4325,
+      "theoretical_loss": 3.480712808692072,
+      "tokens_seen": 1684406272
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000498,
+      "loss": 1.4759,
+      "theoretical_loss": 3.4807011325362307,
+      "tokens_seen": 1684471808
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/aligned_token_count": 0.46138808131217957,
+      "objective/train/avg_token_score": 0.026906196027994156,
+      "objective/train/docs_used": 954676,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.2592566013336182,
+      "objective/train/original_loss": 3.659281015396118,
+      "objective/train/theoretical_loss": 3.4806894569618425,
+      "objective/train/tokens_used": 43396576,
+      "objective/train/unlikelihood_loss": 0.058182328939437866,
+      "theoretical_loss": 3.4806894569618425,
+      "tokens_seen": 1684537344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000497979797979798,
+      "loss": 1.2782,
+      "theoretical_loss": 3.4806894569618425,
+      "tokens_seen": 1684537344
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000497959595959596,
+      "loss": 1.3906,
+      "theoretical_loss": 3.480677781968857,
+      "tokens_seen": 1684602880
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.000497939393939394,
+      "loss": 1.5195,
+      "theoretical_loss": 3.480666107557221,
+      "tokens_seen": 1684668416
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004979191919191919,
+      "loss": 1.494,
+      "theoretical_loss": 3.4806544337268854,
+      "tokens_seen": 1684733952
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978989898989899,
+      "loss": 1.5243,
+      "theoretical_loss": 3.4806427604777967,
+      "tokens_seen": 1684799488
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978787878787879,
+      "loss": 1.3817,
+      "theoretical_loss": 3.4806310878099045,
+      "tokens_seen": 1684865024
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978585858585858,
+      "loss": 1.3818,
+      "theoretical_loss": 3.4806194157231567,
+      "tokens_seen": 1684930560
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978383838383838,
+      "loss": 1.3979,
+      "theoretical_loss": 3.480607744217502,
+      "tokens_seen": 1684996096
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004978181818181818,
+      "loss": 1.5915,
+      "theoretical_loss": 3.4805960732928876,
+      "tokens_seen": 1685061632
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977979797979799,
+      "loss": 1.6397,
+      "theoretical_loss": 3.480584402949264,
+      "tokens_seen": 1685127168
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977777777777778,
+      "loss": 1.7836,
+      "theoretical_loss": 3.4805727331865786,
+      "tokens_seen": 1685192704
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977575757575758,
+      "loss": 1.4281,
+      "theoretical_loss": 3.48056106400478,
+      "tokens_seen": 1685258240
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977373737373738,
+      "loss": 1.5813,
+      "theoretical_loss": 3.480549395403817,
+      "tokens_seen": 1685323776
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004977171717171717,
+      "loss": 1.5063,
+      "theoretical_loss": 3.4805377273836378,
+      "tokens_seen": 1685389312
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976969696969697,
+      "loss": 1.3745,
+      "theoretical_loss": 3.4805260599441903,
+      "tokens_seen": 1685454848
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976767676767677,
+      "loss": 1.5523,
+      "theoretical_loss": 3.480514393085424,
+      "tokens_seen": 1685520384
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976565656565656,
+      "loss": 1.3951,
+      "theoretical_loss": 3.4805027268072877,
+      "tokens_seen": 1685585920
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976363636363636,
+      "loss": 1.3416,
+      "theoretical_loss": 3.4804910611097286,
+      "tokens_seen": 1685651456
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004976161616161616,
+      "loss": 1.5686,
+      "theoretical_loss": 3.4804793959926963,
+      "tokens_seen": 1685716992
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975959595959596,
+      "loss": 1.489,
+      "theoretical_loss": 3.480467731456139,
+      "tokens_seen": 1685782528
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975757575757575,
+      "loss": 1.341,
+      "theoretical_loss": 3.4804560675000054,
+      "tokens_seen": 1685848064
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975555555555556,
+      "loss": 1.2872,
+      "theoretical_loss": 3.4804444041242437,
+      "tokens_seen": 1685913600
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975353535353536,
+      "loss": 1.5454,
+      "theoretical_loss": 3.480432741328803,
+      "tokens_seen": 1685979136
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004975151515151516,
+      "loss": 1.5041,
+      "theoretical_loss": 3.4804210791136314,
+      "tokens_seen": 1686044672
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004974949494949495,
+      "loss": 1.5567,
+      "theoretical_loss": 3.4804094174786773,
+      "tokens_seen": 1686110208
+    },
+    {
+      "epoch": 0.01,
+      "objective/train/aligned_token_count": 0.5329912304878235,
+      "objective/train/avg_token_score": 0.01913226768374443,
+      "objective/train/docs_used": 955358,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.6620068550109863,
+      "objective/train/original_loss": 3.886705160140991,
+      "objective/train/theoretical_loss": 3.4803977564238897,
+      "objective/train/tokens_used": 45034976,
+      "objective/train/unlikelihood_loss": 0.04157567396759987,
+      "theoretical_loss": 3.4803977564238897,
+      "tokens_seen": 1686175744
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 0.0004974747474747475,
+      "loss": 1.5445,
+      "theoretical_loss": 3.4803977564238897,
+      "tokens_seen": 1686175744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974545454545454,
+      "loss": 1.4179,
+      "theoretical_loss": 3.4803860959492177,
+      "tokens_seen": 1686241280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974343434343435,
+      "loss": 1.4412,
+      "theoretical_loss": 3.4803744360546087,
+      "tokens_seen": 1686306816
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004974141414141414,
+      "loss": 1.5498,
+      "theoretical_loss": 3.4803627767400123,
+      "tokens_seen": 1686372352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973939393939394,
+      "loss": 1.4011,
+      "theoretical_loss": 3.480351118005376,
+      "tokens_seen": 1686437888
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973737373737373,
+      "loss": 1.4917,
+      "theoretical_loss": 3.48033945985065,
+      "tokens_seen": 1686503424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973535353535353,
+      "loss": 1.392,
+      "theoretical_loss": 3.4803278022757818,
+      "tokens_seen": 1686568960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973333333333334,
+      "loss": 1.4304,
+      "theoretical_loss": 3.4803161452807196,
+      "tokens_seen": 1686634496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004973131313131314,
+      "loss": 1.4172,
+      "theoretical_loss": 3.4803044888654133,
+      "tokens_seen": 1686700032
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972929292929293,
+      "loss": 1.2836,
+      "theoretical_loss": 3.480292833029811,
+      "tokens_seen": 1686765568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972727272727273,
+      "loss": 1.5495,
+      "theoretical_loss": 3.480281177773861,
+      "tokens_seen": 1686831104
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972525252525253,
+      "loss": 1.3092,
+      "theoretical_loss": 3.4802695230975127,
+      "tokens_seen": 1686896640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972323232323233,
+      "loss": 1.3053,
+      "theoretical_loss": 3.480257869000714,
+      "tokens_seen": 1686962176
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004972121212121212,
+      "loss": 1.3317,
+      "theoretical_loss": 3.4802462154834135,
+      "tokens_seen": 1687027712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971919191919192,
+      "loss": 1.484,
+      "theoretical_loss": 3.4802345625455606,
+      "tokens_seen": 1687093248
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971717171717171,
+      "loss": 1.4059,
+      "theoretical_loss": 3.480222910187104,
+      "tokens_seen": 1687158784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971515151515152,
+      "loss": 1.5274,
+      "theoretical_loss": 3.4802112584079916,
+      "tokens_seen": 1687224320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971313131313131,
+      "loss": 1.1674,
+      "theoretical_loss": 3.4801996072081725,
+      "tokens_seen": 1687289856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004971111111111111,
+      "loss": 1.721,
+      "theoretical_loss": 3.4801879565875957,
+      "tokens_seen": 1687355392
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000497090909090909,
+      "loss": 1.2672,
+      "theoretical_loss": 3.4801763065462095,
+      "tokens_seen": 1687420928
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970707070707071,
+      "loss": 1.6776,
+      "theoretical_loss": 3.4801646570839626,
+      "tokens_seen": 1687486464
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970505050505051,
+      "loss": 1.3374,
+      "theoretical_loss": 3.480153008200804,
+      "tokens_seen": 1687552000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004970303030303031,
+      "loss": 1.2859,
+      "theoretical_loss": 3.480141359896683,
+      "tokens_seen": 1687617536
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000497010101010101,
+      "loss": 1.5063,
+      "theoretical_loss": 3.4801297121715464,
+      "tokens_seen": 1687683072
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496989898989899,
+      "loss": 1.3917,
+      "theoretical_loss": 3.4801180650253447,
+      "tokens_seen": 1687748608
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/aligned_token_count": 0.4268084168434143,
+      "objective/train/avg_token_score": 0.010676675476133823,
+      "objective/train/docs_used": 956671,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.4835927486419678,
+      "objective/train/original_loss": 4.212736129760742,
+      "objective/train/theoretical_loss": 3.480106418458026,
+      "objective/train/tokens_used": 46673376,
+      "objective/train/unlikelihood_loss": 0.08470705896615982,
+      "theoretical_loss": 3.480106418458026,
+      "tokens_seen": 1687814144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496969696969697,
+      "loss": 1.4357,
+      "theoretical_loss": 3.480106418458026,
+      "tokens_seen": 1687814144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496949494949495,
+      "loss": 1.3479,
+      "theoretical_loss": 3.4800947724695392,
+      "tokens_seen": 1687879680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969292929292929,
+      "loss": 1.2908,
+      "theoretical_loss": 3.4800831270598334,
+      "tokens_seen": 1687945216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004969090909090909,
+      "loss": 1.506,
+      "theoretical_loss": 3.4800714822288565,
+      "tokens_seen": 1688010752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968888888888889,
+      "loss": 1.5604,
+      "theoretical_loss": 3.4800598379765577,
+      "tokens_seen": 1688076288
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968686868686868,
+      "loss": 1.435,
+      "theoretical_loss": 3.4800481943028863,
+      "tokens_seen": 1688141824
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968484848484849,
+      "loss": 1.3855,
+      "theoretical_loss": 3.48003655120779,
+      "tokens_seen": 1688207360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968282828282829,
+      "loss": 1.579,
+      "theoretical_loss": 3.4800249086912185,
+      "tokens_seen": 1688272896
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004968080808080808,
+      "loss": 1.5069,
+      "theoretical_loss": 3.4800132667531205,
+      "tokens_seen": 1688338432
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967878787878788,
+      "loss": 1.5546,
+      "theoretical_loss": 3.4800016253934447,
+      "tokens_seen": 1688403968
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967676767676768,
+      "loss": 1.2675,
+      "theoretical_loss": 3.4799899846121396,
+      "tokens_seen": 1688469504
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967474747474748,
+      "loss": 1.5451,
+      "theoretical_loss": 3.4799783444091545,
+      "tokens_seen": 1688535040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967272727272727,
+      "loss": 1.4007,
+      "theoretical_loss": 3.4799667047844376,
+      "tokens_seen": 1688600576
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004967070707070707,
+      "loss": 1.4922,
+      "theoretical_loss": 3.479955065737938,
+      "tokens_seen": 1688666112
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966868686868687,
+      "loss": 1.6596,
+      "theoretical_loss": 3.479943427269605,
+      "tokens_seen": 1688731648
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966666666666666,
+      "loss": 1.7516,
+      "theoretical_loss": 3.4799317893793873,
+      "tokens_seen": 1688797184
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966464646464646,
+      "loss": 1.437,
+      "theoretical_loss": 3.479920152067233,
+      "tokens_seen": 1688862720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966262626262627,
+      "loss": 1.3793,
+      "theoretical_loss": 3.4799085153330918,
+      "tokens_seen": 1688928256
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004966060606060607,
+      "loss": 1.6005,
+      "theoretical_loss": 3.479896879176912,
+      "tokens_seen": 1688993792
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965858585858586,
+      "loss": 1.5023,
+      "theoretical_loss": 3.479885243598643,
+      "tokens_seen": 1689059328
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965656565656566,
+      "loss": 1.4687,
+      "theoretical_loss": 3.4798736085982336,
+      "tokens_seen": 1689124864
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965454545454546,
+      "loss": 1.6716,
+      "theoretical_loss": 3.479861974175632,
+      "tokens_seen": 1689190400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965252525252525,
+      "loss": 1.6124,
+      "theoretical_loss": 3.4798503403307874,
+      "tokens_seen": 1689255936
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004965050505050505,
+      "loss": 1.2172,
+      "theoretical_loss": 3.4798387070636494,
+      "tokens_seen": 1689321472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964848484848485,
+      "loss": 1.7067,
+      "theoretical_loss": 3.479827074374166,
+      "tokens_seen": 1689387008
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/aligned_token_count": 0.4199046790599823,
+      "objective/train/avg_token_score": 0.011589493602514267,
+      "objective/train/docs_used": 957370,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.3580185174942017,
+      "objective/train/original_loss": 4.241112232208252,
+      "objective/train/theoretical_loss": 3.4798154422622867,
+      "objective/train/tokens_used": 48311776,
+      "objective/train/unlikelihood_loss": 0.06464245170354843,
+      "theoretical_loss": 3.4798154422622867,
+      "tokens_seen": 1689452544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964646464646464,
+      "loss": 1.4656,
+      "theoretical_loss": 3.4798154422622867,
+      "tokens_seen": 1689452544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964444444444444,
+      "loss": 1.4522,
+      "theoretical_loss": 3.4798038107279603,
+      "tokens_seen": 1689518080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964242424242424,
+      "loss": 1.5003,
+      "theoretical_loss": 3.4797921797711355,
+      "tokens_seen": 1689583616
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004964040404040405,
+      "loss": 1.4859,
+      "theoretical_loss": 3.4797805493917613,
+      "tokens_seen": 1689649152
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963838383838384,
+      "loss": 1.5211,
+      "theoretical_loss": 3.4797689195897865,
+      "tokens_seen": 1689714688
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963636363636364,
+      "loss": 1.5367,
+      "theoretical_loss": 3.4797572903651606,
+      "tokens_seen": 1689780224
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963434343434344,
+      "loss": 1.4631,
+      "theoretical_loss": 3.479745661717832,
+      "tokens_seen": 1689845760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963232323232324,
+      "loss": 1.4319,
+      "theoretical_loss": 3.47973403364775,
+      "tokens_seen": 1689911296
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004963030303030303,
+      "loss": 1.2408,
+      "theoretical_loss": 3.4797224061548633,
+      "tokens_seen": 1689976832
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962828282828283,
+      "loss": 1.3776,
+      "theoretical_loss": 3.4797107792391206,
+      "tokens_seen": 1690042368
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962626262626262,
+      "loss": 1.4295,
+      "theoretical_loss": 3.4796991529004715,
+      "tokens_seen": 1690107904
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962424242424243,
+      "loss": 1.5602,
+      "theoretical_loss": 3.4796875271388648,
+      "tokens_seen": 1690173440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962222222222222,
+      "loss": 1.3299,
+      "theoretical_loss": 3.4796759019542494,
+      "tokens_seen": 1690238976
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004962020202020202,
+      "loss": 1.4424,
+      "theoretical_loss": 3.479664277346574,
+      "tokens_seen": 1690304512
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961818181818181,
+      "loss": 1.6119,
+      "theoretical_loss": 3.479652653315788,
+      "tokens_seen": 1690370048
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961616161616161,
+      "loss": 1.5685,
+      "theoretical_loss": 3.4796410298618405,
+      "tokens_seen": 1690435584
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961414141414142,
+      "loss": 1.4242,
+      "theoretical_loss": 3.4796294069846803,
+      "tokens_seen": 1690501120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961212121212122,
+      "loss": 1.4835,
+      "theoretical_loss": 3.4796177846842564,
+      "tokens_seen": 1690566656
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004961010101010101,
+      "loss": 1.3436,
+      "theoretical_loss": 3.479606162960518,
+      "tokens_seen": 1690632192
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960808080808081,
+      "loss": 1.6164,
+      "theoretical_loss": 3.4795945418134133,
+      "tokens_seen": 1690697728
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960606060606061,
+      "loss": 1.6758,
+      "theoretical_loss": 3.479582921242893,
+      "tokens_seen": 1690763264
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004960404040404041,
+      "loss": 1.298,
+      "theoretical_loss": 3.4795713012489045,
+      "tokens_seen": 1690828800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496020202020202,
+      "loss": 1.4461,
+      "theoretical_loss": 3.4795596818313976,
+      "tokens_seen": 1690894336
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000496,
+      "loss": 1.4362,
+      "theoretical_loss": 3.4795480629903217,
+      "tokens_seen": 1690959872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959797979797979,
+      "loss": 1.3436,
+      "theoretical_loss": 3.479536444725625,
+      "tokens_seen": 1691025408
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/aligned_token_count": 0.38599705696105957,
+      "objective/train/avg_token_score": 0.048682406544685364,
+      "objective/train/docs_used": 958661,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.2436673641204834,
+      "objective/train/original_loss": 4.153236389160156,
+      "objective/train/theoretical_loss": 3.4795248270372574,
+      "objective/train/tokens_used": 49950176,
+      "objective/train/unlikelihood_loss": 0.07140035182237625,
+      "theoretical_loss": 3.4795248270372574,
+      "tokens_seen": 1691090944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004959595959595959,
+      "loss": 1.2475,
+      "theoretical_loss": 3.4795248270372574,
+      "tokens_seen": 1691090944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495939393939394,
+      "loss": 1.6063,
+      "theoretical_loss": 3.479513209925167,
+      "tokens_seen": 1691156480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495919191919192,
+      "loss": 1.4383,
+      "theoretical_loss": 3.479501593389304,
+      "tokens_seen": 1691222016
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958989898989899,
+      "loss": 1.6435,
+      "theoretical_loss": 3.479489977429617,
+      "tokens_seen": 1691287552
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958787878787879,
+      "loss": 1.5405,
+      "theoretical_loss": 3.4794783620460548,
+      "tokens_seen": 1691353088
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958585858585859,
+      "loss": 1.5615,
+      "theoretical_loss": 3.479466747238567,
+      "tokens_seen": 1691418624
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958383838383839,
+      "loss": 1.51,
+      "theoretical_loss": 3.4794551330071024,
+      "tokens_seen": 1691484160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004958181818181818,
+      "loss": 1.4175,
+      "theoretical_loss": 3.47944351935161,
+      "tokens_seen": 1691549696
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957979797979798,
+      "loss": 1.5496,
+      "theoretical_loss": 3.4794319062720396,
+      "tokens_seen": 1691615232
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957777777777778,
+      "loss": 1.4502,
+      "theoretical_loss": 3.479420293768339,
+      "tokens_seen": 1691680768
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957575757575757,
+      "loss": 1.4327,
+      "theoretical_loss": 3.479408681840459,
+      "tokens_seen": 1691746304
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957373737373737,
+      "loss": 1.4366,
+      "theoretical_loss": 3.4793970704883477,
+      "tokens_seen": 1691811840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004957171717171717,
+      "loss": 1.2897,
+      "theoretical_loss": 3.4793854597119545,
+      "tokens_seen": 1691877376
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956969696969698,
+      "loss": 1.6798,
+      "theoretical_loss": 3.4793738495112283,
+      "tokens_seen": 1691942912
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956767676767677,
+      "loss": 1.2665,
+      "theoretical_loss": 3.4793622398861186,
+      "tokens_seen": 1692008448
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956565656565657,
+      "loss": 1.5467,
+      "theoretical_loss": 3.4793506308365747,
+      "tokens_seen": 1692073984
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956363636363637,
+      "loss": 1.3853,
+      "theoretical_loss": 3.479339022362545,
+      "tokens_seen": 1692139520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004956161616161616,
+      "loss": 1.5345,
+      "theoretical_loss": 3.47932741446398,
+      "tokens_seen": 1692205056
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955959595959596,
+      "loss": 1.6217,
+      "theoretical_loss": 3.4793158071408277,
+      "tokens_seen": 1692270592
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955757575757576,
+      "loss": 1.6179,
+      "theoretical_loss": 3.4793042003930377,
+      "tokens_seen": 1692336128
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955555555555556,
+      "loss": 1.3264,
+      "theoretical_loss": 3.479292594220559,
+      "tokens_seen": 1692401664
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955353535353535,
+      "loss": 1.6683,
+      "theoretical_loss": 3.4792809886233416,
+      "tokens_seen": 1692467200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004955151515151515,
+      "loss": 1.6922,
+      "theoretical_loss": 3.4792693836013333,
+      "tokens_seen": 1692532736
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954949494949495,
+      "loss": 1.6503,
+      "theoretical_loss": 3.4792577791544845,
+      "tokens_seen": 1692598272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954747474747474,
+      "loss": 1.6034,
+      "theoretical_loss": 3.4792461752827446,
+      "tokens_seen": 1692663808
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/aligned_token_count": 0.4952957034111023,
+      "objective/train/avg_token_score": 0.008989947848021984,
+      "objective/train/docs_used": 959159,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.6178122758865356,
+      "objective/train/original_loss": 3.7790725231170654,
+      "objective/train/theoretical_loss": 3.479234571986062,
+      "objective/train/tokens_used": 51588576,
+      "objective/train/unlikelihood_loss": 0.05506987124681473,
+      "theoretical_loss": 3.479234571986062,
+      "tokens_seen": 1692729344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954545454545455,
+      "loss": 1.5392,
+      "theoretical_loss": 3.479234571986062,
+      "tokens_seen": 1692729344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954343434343435,
+      "loss": 1.5108,
+      "theoretical_loss": 3.479222969264386,
+      "tokens_seen": 1692794880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004954141414141415,
+      "loss": 1.404,
+      "theoretical_loss": 3.4792113671176663,
+      "tokens_seen": 1692860416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953939393939394,
+      "loss": 1.4155,
+      "theoretical_loss": 3.479199765545852,
+      "tokens_seen": 1692925952
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953737373737374,
+      "loss": 1.4939,
+      "theoretical_loss": 3.4791881645488916,
+      "tokens_seen": 1692991488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953535353535354,
+      "loss": 1.5546,
+      "theoretical_loss": 3.479176564126736,
+      "tokens_seen": 1693057024
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953333333333334,
+      "loss": 1.6032,
+      "theoretical_loss": 3.4791649642793328,
+      "tokens_seen": 1693122560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004953131313131313,
+      "loss": 1.4701,
+      "theoretical_loss": 3.479153365006632,
+      "tokens_seen": 1693188096
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952929292929293,
+      "loss": 1.4542,
+      "theoretical_loss": 3.479141766308584,
+      "tokens_seen": 1693253632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952727272727272,
+      "loss": 1.508,
+      "theoretical_loss": 3.479130168185136,
+      "tokens_seen": 1693319168
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952525252525252,
+      "loss": 1.5405,
+      "theoretical_loss": 3.4791185706362384,
+      "tokens_seen": 1693384704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952323232323232,
+      "loss": 1.3952,
+      "theoretical_loss": 3.4791069736618407,
+      "tokens_seen": 1693450240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004952121212121213,
+      "loss": 1.4216,
+      "theoretical_loss": 3.479095377261892,
+      "tokens_seen": 1693515776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951919191919192,
+      "loss": 1.5664,
+      "theoretical_loss": 3.4790837814363407,
+      "tokens_seen": 1693581312
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951717171717172,
+      "loss": 1.6256,
+      "theoretical_loss": 3.479072186185138,
+      "tokens_seen": 1693646848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951515151515152,
+      "loss": 1.3279,
+      "theoretical_loss": 3.4790605915082313,
+      "tokens_seen": 1693712384
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951313131313132,
+      "loss": 1.3493,
+      "theoretical_loss": 3.479048997405571,
+      "tokens_seen": 1693777920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004951111111111111,
+      "loss": 1.6102,
+      "theoretical_loss": 3.479037403877107,
+      "tokens_seen": 1693843456
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950909090909091,
+      "loss": 1.2949,
+      "theoretical_loss": 3.4790258109227867,
+      "tokens_seen": 1693908992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495070707070707,
+      "loss": 1.3058,
+      "theoretical_loss": 3.479014218542561,
+      "tokens_seen": 1693974528
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004950505050505051,
+      "loss": 1.375,
+      "theoretical_loss": 3.4790026267363796,
+      "tokens_seen": 1694040064
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495030303030303,
+      "loss": 1.4453,
+      "theoretical_loss": 3.478991035504191,
+      "tokens_seen": 1694105600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000495010101010101,
+      "loss": 1.55,
+      "theoretical_loss": 3.4789794448459443,
+      "tokens_seen": 1694171136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494989898989899,
+      "loss": 1.5699,
+      "theoretical_loss": 3.4789678547615894,
+      "tokens_seen": 1694236672
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494969696969697,
+      "loss": 1.3551,
+      "theoretical_loss": 3.478956265251076,
+      "tokens_seen": 1694302208
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/aligned_token_count": 0.3170210123062134,
+      "objective/train/avg_token_score": 0.017827244475483894,
+      "objective/train/docs_used": 960536,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 0.9088751673698425,
+      "objective/train/original_loss": 4.046257972717285,
+      "objective/train/theoretical_loss": 3.478944676314353,
+      "objective/train/tokens_used": 53226976,
+      "objective/train/unlikelihood_loss": 0.10128523409366608,
+      "theoretical_loss": 3.478944676314353,
+      "tokens_seen": 1694367744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494949494949495,
+      "loss": 1.3764,
+      "theoretical_loss": 3.478944676314353,
+      "tokens_seen": 1694367744
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494929292929293,
+      "loss": 1.7527,
+      "theoretical_loss": 3.4789330879513694,
+      "tokens_seen": 1694433280
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004949090909090909,
+      "loss": 1.5846,
+      "theoretical_loss": 3.478921500162076,
+      "tokens_seen": 1694498816
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948888888888889,
+      "loss": 1.3229,
+      "theoretical_loss": 3.4789099129464205,
+      "tokens_seen": 1694564352
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948686868686869,
+      "loss": 1.5288,
+      "theoretical_loss": 3.4788983263043534,
+      "tokens_seen": 1694629888
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948484848484849,
+      "loss": 1.5913,
+      "theoretical_loss": 3.478886740235824,
+      "tokens_seen": 1694695424
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948282828282828,
+      "loss": 1.4224,
+      "theoretical_loss": 3.4788751547407815,
+      "tokens_seen": 1694760960
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004948080808080808,
+      "loss": 1.6395,
+      "theoretical_loss": 3.478863569819176,
+      "tokens_seen": 1694826496
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947878787878787,
+      "loss": 1.4925,
+      "theoretical_loss": 3.478851985470956,
+      "tokens_seen": 1694892032
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947676767676767,
+      "loss": 1.5345,
+      "theoretical_loss": 3.478840401696071,
+      "tokens_seen": 1694957568
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947474747474748,
+      "loss": 1.3046,
+      "theoretical_loss": 3.4788288184944713,
+      "tokens_seen": 1695023104
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947272727272728,
+      "loss": 1.5731,
+      "theoretical_loss": 3.4788172358661056,
+      "tokens_seen": 1695088640
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004947070707070707,
+      "loss": 1.3865,
+      "theoretical_loss": 3.478805653810924,
+      "tokens_seen": 1695154176
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946868686868687,
+      "loss": 1.3987,
+      "theoretical_loss": 3.478794072328875,
+      "tokens_seen": 1695219712
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946666666666667,
+      "loss": 1.4943,
+      "theoretical_loss": 3.478782491419909,
+      "tokens_seen": 1695285248
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946464646464647,
+      "loss": 1.2217,
+      "theoretical_loss": 3.4787709110839753,
+      "tokens_seen": 1695350784
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946262626262626,
+      "loss": 1.4646,
+      "theoretical_loss": 3.478759331321023,
+      "tokens_seen": 1695416320
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004946060606060606,
+      "loss": 1.4741,
+      "theoretical_loss": 3.478747752131002,
+      "tokens_seen": 1695481856
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945858585858586,
+      "loss": 1.4813,
+      "theoretical_loss": 3.4787361735138624,
+      "tokens_seen": 1695547392
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945656565656565,
+      "loss": 1.3267,
+      "theoretical_loss": 3.478724595469552,
+      "tokens_seen": 1695612928
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945454545454545,
+      "loss": 1.3645,
+      "theoretical_loss": 3.4787130179980217,
+      "tokens_seen": 1695678464
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945252525252526,
+      "loss": 1.4331,
+      "theoretical_loss": 3.478701441099221,
+      "tokens_seen": 1695744000
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004945050505050506,
+      "loss": 1.481,
+      "theoretical_loss": 3.4786898647730986,
+      "tokens_seen": 1695809536
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944848484848485,
+      "loss": 1.4608,
+      "theoretical_loss": 3.478678289019605,
+      "tokens_seen": 1695875072
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944646464646465,
+      "loss": 1.4611,
+      "theoretical_loss": 3.478666713838689,
+      "tokens_seen": 1695940608
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/aligned_token_count": 0.3928396999835968,
+      "objective/train/avg_token_score": 0.021624328568577766,
+      "objective/train/docs_used": 961177,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.3411294221878052,
+      "objective/train/original_loss": 4.00459623336792,
+      "objective/train/theoretical_loss": 3.478655139230301,
+      "objective/train/tokens_used": 54865376,
+      "objective/train/unlikelihood_loss": 0.05852148309350014,
+      "theoretical_loss": 3.478655139230301,
+      "tokens_seen": 1696006144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944444444444445,
+      "loss": 1.4319,
+      "theoretical_loss": 3.478655139230301,
+      "tokens_seen": 1696006144
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944242424242424,
+      "loss": 1.432,
+      "theoretical_loss": 3.478643565194389,
+      "tokens_seen": 1696071680
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004944040404040404,
+      "loss": 1.4987,
+      "theoretical_loss": 3.478631991730904,
+      "tokens_seen": 1696137216
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943838383838384,
+      "loss": 1.6415,
+      "theoretical_loss": 3.478620418839795,
+      "tokens_seen": 1696202752
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943636363636363,
+      "loss": 1.6562,
+      "theoretical_loss": 3.4786088465210123,
+      "tokens_seen": 1696268288
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943434343434343,
+      "loss": 1.3873,
+      "theoretical_loss": 3.478597274774504,
+      "tokens_seen": 1696333824
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943232323232323,
+      "loss": 1.3246,
+      "theoretical_loss": 3.4785857036002215,
+      "tokens_seen": 1696399360
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004943030303030303,
+      "loss": 1.3636,
+      "theoretical_loss": 3.478574132998113,
+      "tokens_seen": 1696464896
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942828282828283,
+      "loss": 1.446,
+      "theoretical_loss": 3.478562562968129,
+      "tokens_seen": 1696530432
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942626262626263,
+      "loss": 1.3962,
+      "theoretical_loss": 3.4785509935102183,
+      "tokens_seen": 1696595968
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942424242424243,
+      "loss": 1.4457,
+      "theoretical_loss": 3.4785394246243317,
+      "tokens_seen": 1696661504
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942222222222223,
+      "loss": 1.6089,
+      "theoretical_loss": 3.478527856310417,
+      "tokens_seen": 1696727040
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004942020202020202,
+      "loss": 1.5574,
+      "theoretical_loss": 3.4785162885684255,
+      "tokens_seen": 1696792576
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941818181818182,
+      "loss": 1.4105,
+      "theoretical_loss": 3.478504721398306,
+      "tokens_seen": 1696858112
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941616161616161,
+      "loss": 1.399,
+      "theoretical_loss": 3.4784931548000086,
+      "tokens_seen": 1696923648
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941414141414142,
+      "loss": 1.5323,
+      "theoretical_loss": 3.4784815887734823,
+      "tokens_seen": 1696989184
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941212121212121,
+      "loss": 1.5829,
+      "theoretical_loss": 3.4784700233186774,
+      "tokens_seen": 1697054720
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004941010101010101,
+      "loss": 1.4786,
+      "theoretical_loss": 3.4784584584355436,
+      "tokens_seen": 1697120256
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494080808080808,
+      "loss": 1.5934,
+      "theoretical_loss": 3.47844689412403,
+      "tokens_seen": 1697185792
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494060606060606,
+      "loss": 1.4969,
+      "theoretical_loss": 3.4784353303840865,
+      "tokens_seen": 1697251328
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940404040404041,
+      "loss": 1.5708,
+      "theoretical_loss": 3.478423767215663,
+      "tokens_seen": 1697316864
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004940202020202021,
+      "loss": 1.3845,
+      "theoretical_loss": 3.478412204618709,
+      "tokens_seen": 1697382400
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000494,
+      "loss": 1.3359,
+      "theoretical_loss": 3.4784006425931744,
+      "tokens_seen": 1697447936
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493979797979798,
+      "loss": 1.4839,
+      "theoretical_loss": 3.478389081139009,
+      "tokens_seen": 1697513472
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493959595959596,
+      "loss": 1.5483,
+      "theoretical_loss": 3.4783775202561618,
+      "tokens_seen": 1697579008
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/aligned_token_count": 0.5273093581199646,
+      "objective/train/avg_token_score": 0.005273816175758839,
+      "objective/train/docs_used": 961828,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.706133484840393,
+      "objective/train/original_loss": 3.877432107925415,
+      "objective/train/theoretical_loss": 3.4783659599445826,
+      "objective/train/tokens_used": 56503776,
+      "objective/train/unlikelihood_loss": 0.0712694600224495,
+      "theoretical_loss": 3.4783659599445826,
+      "tokens_seen": 1697644544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493939393939394,
+      "loss": 1.5571,
+      "theoretical_loss": 3.4783659599445826,
+      "tokens_seen": 1697644544
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004939191919191919,
+      "loss": 1.2926,
+      "theoretical_loss": 3.478354400204222,
+      "tokens_seen": 1697710080
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938989898989899,
+      "loss": 1.3542,
+      "theoretical_loss": 3.4783428410350297,
+      "tokens_seen": 1697775616
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938787878787878,
+      "loss": 1.6162,
+      "theoretical_loss": 3.4783312824369537,
+      "tokens_seen": 1697841152
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938585858585859,
+      "loss": 1.5232,
+      "theoretical_loss": 3.478319724409946,
+      "tokens_seen": 1697906688
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938383838383838,
+      "loss": 1.5115,
+      "theoretical_loss": 3.478308166953955,
+      "tokens_seen": 1697972224
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004938181818181819,
+      "loss": 1.6203,
+      "theoretical_loss": 3.4782966100689308,
+      "tokens_seen": 1698037760
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937979797979798,
+      "loss": 1.6335,
+      "theoretical_loss": 3.4782850537548233,
+      "tokens_seen": 1698103296
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937777777777778,
+      "loss": 1.3283,
+      "theoretical_loss": 3.478273498011582,
+      "tokens_seen": 1698168832
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937575757575758,
+      "loss": 1.5045,
+      "theoretical_loss": 3.4782619428391564,
+      "tokens_seen": 1698234368
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937373737373738,
+      "loss": 1.5789,
+      "theoretical_loss": 3.478250388237497,
+      "tokens_seen": 1698299904
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004937171717171717,
+      "loss": 1.4186,
+      "theoretical_loss": 3.478238834206553,
+      "tokens_seen": 1698365440
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936969696969697,
+      "loss": 1.4197,
+      "theoretical_loss": 3.4782272807462746,
+      "tokens_seen": 1698430976
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936767676767677,
+      "loss": 1.384,
+      "theoretical_loss": 3.4782157278566115,
+      "tokens_seen": 1698496512
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936565656565657,
+      "loss": 1.553,
+      "theoretical_loss": 3.4782041755375133,
+      "tokens_seen": 1698562048
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936363636363636,
+      "loss": 1.4178,
+      "theoretical_loss": 3.47819262378893,
+      "tokens_seen": 1698627584
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004936161616161616,
+      "loss": 1.6003,
+      "theoretical_loss": 3.478181072610811,
+      "tokens_seen": 1698693120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935959595959597,
+      "loss": 1.0584,
+      "theoretical_loss": 3.478169522003107,
+      "tokens_seen": 1698758656
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935757575757576,
+      "loss": 1.4205,
+      "theoretical_loss": 3.478157971965767,
+      "tokens_seen": 1698824192
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935555555555556,
+      "loss": 1.4223,
+      "theoretical_loss": 3.4781464224987415,
+      "tokens_seen": 1698889728
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935353535353536,
+      "loss": 1.3599,
+      "theoretical_loss": 3.478134873601979,
+      "tokens_seen": 1698955264
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004935151515151515,
+      "loss": 1.375,
+      "theoretical_loss": 3.4781233252754316,
+      "tokens_seen": 1699020800
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934949494949495,
+      "loss": 1.4275,
+      "theoretical_loss": 3.4781117775190467,
+      "tokens_seen": 1699086336
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934747474747475,
+      "loss": 1.3522,
+      "theoretical_loss": 3.4781002303327764,
+      "tokens_seen": 1699151872
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934545454545455,
+      "loss": 1.48,
+      "theoretical_loss": 3.4780886837165688,
+      "tokens_seen": 1699217408
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/aligned_token_count": 0.35807672142982483,
+      "objective/train/avg_token_score": 0.012293923646211624,
+      "objective/train/docs_used": 963095,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.1824865341186523,
+      "objective/train/original_loss": 4.2231035232543945,
+      "objective/train/theoretical_loss": 3.4780771376703745,
+      "objective/train/tokens_used": 58142176,
+      "objective/train/unlikelihood_loss": 0.0799039676785469,
+      "theoretical_loss": 3.4780771376703745,
+      "tokens_seen": 1699282944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934343434343434,
+      "loss": 1.2993,
+      "theoretical_loss": 3.4780771376703745,
+      "tokens_seen": 1699282944
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004934141414141414,
+      "loss": 1.4704,
+      "theoretical_loss": 3.4780655921941435,
+      "tokens_seen": 1699348480
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933939393939394,
+      "loss": 1.3564,
+      "theoretical_loss": 3.4780540472878254,
+      "tokens_seen": 1699414016
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933737373737373,
+      "loss": 1.487,
+      "theoretical_loss": 3.47804250295137,
+      "tokens_seen": 1699479552
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933535353535354,
+      "loss": 1.331,
+      "theoretical_loss": 3.478030959184728,
+      "tokens_seen": 1699545088
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933333333333334,
+      "loss": 1.6139,
+      "theoretical_loss": 3.4780194159878484,
+      "tokens_seen": 1699610624
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004933131313131314,
+      "loss": 1.4798,
+      "theoretical_loss": 3.4780078733606814,
+      "tokens_seen": 1699676160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932929292929293,
+      "loss": 1.516,
+      "theoretical_loss": 3.477996331303177,
+      "tokens_seen": 1699741696
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932727272727273,
+      "loss": 1.5999,
+      "theoretical_loss": 3.4779847898152854,
+      "tokens_seen": 1699807232
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932525252525253,
+      "loss": 1.4377,
+      "theoretical_loss": 3.477973248896956,
+      "tokens_seen": 1699872768
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932323232323232,
+      "loss": 1.4555,
+      "theoretical_loss": 3.4779617085481385,
+      "tokens_seen": 1699938304
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004932121212121212,
+      "loss": 1.2517,
+      "theoretical_loss": 3.4779501687687837,
+      "tokens_seen": 1700003840
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931919191919192,
+      "loss": 1.5707,
+      "theoretical_loss": 3.477938629558841,
+      "tokens_seen": 1700069376
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931717171717171,
+      "loss": 1.4201,
+      "theoretical_loss": 3.4779270909182607,
+      "tokens_seen": 1700134912
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931515151515151,
+      "loss": 1.539,
+      "theoretical_loss": 3.4779155528469925,
+      "tokens_seen": 1700200448
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931313131313131,
+      "loss": 1.4055,
+      "theoretical_loss": 3.477904015344986,
+      "tokens_seen": 1700265984
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004931111111111112,
+      "loss": 1.628,
+      "theoretical_loss": 3.4778924784121914,
+      "tokens_seen": 1700331520
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930909090909091,
+      "loss": 1.4366,
+      "theoretical_loss": 3.47788094204856,
+      "tokens_seen": 1700397056
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930707070707071,
+      "loss": 1.5349,
+      "theoretical_loss": 3.4778694062540394,
+      "tokens_seen": 1700462592
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930505050505051,
+      "loss": 1.335,
+      "theoretical_loss": 3.4778578710285815,
+      "tokens_seen": 1700528128
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004930303030303031,
+      "loss": 1.4479,
+      "theoretical_loss": 3.477846336372136,
+      "tokens_seen": 1700593664
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000493010101010101,
+      "loss": 1.6763,
+      "theoretical_loss": 3.4778348022846517,
+      "tokens_seen": 1700659200
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000492989898989899,
+      "loss": 1.5868,
+      "theoretical_loss": 3.47782326876608,
+      "tokens_seen": 1700724736
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929696969696969,
+      "loss": 1.4506,
+      "theoretical_loss": 3.4778117358163705,
+      "tokens_seen": 1700790272
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.000492949494949495,
+      "loss": 1.5949,
+      "theoretical_loss": 3.4778002034354722,
+      "tokens_seen": 1700855808
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/aligned_token_count": 0.4352394938468933,
+      "objective/train/avg_token_score": 0.022302182391285896,
+      "objective/train/docs_used": 963811,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.3806675672531128,
+      "objective/train/original_loss": 3.8218584060668945,
+      "objective/train/theoretical_loss": 3.477788671623337,
+      "objective/train/tokens_used": 59780576,
+      "objective/train/unlikelihood_loss": 0.0842813178896904,
+      "theoretical_loss": 3.477788671623337,
+      "tokens_seen": 1700921344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929292929292929,
+      "loss": 1.6276,
+      "theoretical_loss": 3.477788671623337,
+      "tokens_seen": 1700921344
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004929090909090909,
+      "loss": 1.3813,
+      "theoretical_loss": 3.4777771403799136,
+      "tokens_seen": 1700986880
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928888888888888,
+      "loss": 1.6226,
+      "theoretical_loss": 3.4777656097051524,
+      "tokens_seen": 1701052416
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928686868686869,
+      "loss": 1.6444,
+      "theoretical_loss": 3.4777540795990034,
+      "tokens_seen": 1701117952
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928484848484849,
+      "loss": 1.3509,
+      "theoretical_loss": 3.477742550061417,
+      "tokens_seen": 1701183488
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928282828282829,
+      "loss": 1.2636,
+      "theoretical_loss": 3.4777310210923424,
+      "tokens_seen": 1701249024
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004928080808080808,
+      "loss": 1.486,
+      "theoretical_loss": 3.4777194926917305,
+      "tokens_seen": 1701314560
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927878787878788,
+      "loss": 1.339,
+      "theoretical_loss": 3.4777079648595315,
+      "tokens_seen": 1701380096
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927676767676768,
+      "loss": 1.5191,
+      "theoretical_loss": 3.4776964375956947,
+      "tokens_seen": 1701445632
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927474747474748,
+      "loss": 1.4387,
+      "theoretical_loss": 3.477684910900171,
+      "tokens_seen": 1701511168
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927272727272727,
+      "loss": 1.3134,
+      "theoretical_loss": 3.4776733847729098,
+      "tokens_seen": 1701576704
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004927070707070707,
+      "loss": 1.4071,
+      "theoretical_loss": 3.4776618592138613,
+      "tokens_seen": 1701642240
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926868686868686,
+      "loss": 1.3664,
+      "theoretical_loss": 3.477650334222976,
+      "tokens_seen": 1701707776
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926666666666666,
+      "loss": 1.5606,
+      "theoretical_loss": 3.477638809800204,
+      "tokens_seen": 1701773312
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926464646464647,
+      "loss": 1.4081,
+      "theoretical_loss": 3.477627285945495,
+      "tokens_seen": 1701838848
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926262626262627,
+      "loss": 1.5566,
+      "theoretical_loss": 3.4776157626587993,
+      "tokens_seen": 1701904384
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004926060606060606,
+      "loss": 1.3377,
+      "theoretical_loss": 3.4776042399400673,
+      "tokens_seen": 1701969920
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925858585858586,
+      "loss": 1.1917,
+      "theoretical_loss": 3.477592717789248,
+      "tokens_seen": 1702035456
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925656565656566,
+      "loss": 1.3057,
+      "theoretical_loss": 3.4775811962062937,
+      "tokens_seen": 1702100992
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925454545454546,
+      "loss": 1.4049,
+      "theoretical_loss": 3.4775696751911527,
+      "tokens_seen": 1702166528
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925252525252525,
+      "loss": 1.5844,
+      "theoretical_loss": 3.4775581547437757,
+      "tokens_seen": 1702232064
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004925050505050505,
+      "loss": 1.5742,
+      "theoretical_loss": 3.477546634864113,
+      "tokens_seen": 1702297600
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924848484848485,
+      "loss": 1.6743,
+      "theoretical_loss": 3.477535115552115,
+      "tokens_seen": 1702363136
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924646464646464,
+      "loss": 1.7692,
+      "theoretical_loss": 3.4775235968077314,
+      "tokens_seen": 1702428672
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.0004924444444444444,
+      "loss": 1.4189,
+      "theoretical_loss": 3.4775120786309124,
+      "tokens_seen": 1702494208
+    },
+    {
+      "epoch": 0.02,
+      "objective/train/aligned_token_count": 0.4039589464664459,
+      "objective/train/avg_token_score": 0.022879842668771744,
+      "objective/train/docs_used": 965136,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.2613023519515991,
+      "objective/train/original_loss": 4.1900811195373535,
+      "objective/train/theoretical_loss": 3.4775005610216088,
+      "objective/train/tokens_used": 61418976,
+      "objective/train/unlikelihood_loss": 0.07926816493272781,
+      "theoretical_loss": 3.4775005610216088,
+      "tokens_seen": 1702559744
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004924242424242425,
+      "loss": 1.3197,
+      "theoretical_loss": 3.4775005610216088,
+      "tokens_seen": 1702559744
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004924040404040405,
+      "loss": 1.4524,
+      "theoretical_loss": 3.4774890439797694,
+      "tokens_seen": 1702625280
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923838383838384,
+      "loss": 1.4616,
+      "theoretical_loss": 3.4774775275053464,
+      "tokens_seen": 1702690816
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923636363636364,
+      "loss": 1.4173,
+      "theoretical_loss": 3.4774660115982883,
+      "tokens_seen": 1702756352
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923434343434344,
+      "loss": 1.4959,
+      "theoretical_loss": 3.477454496258546,
+      "tokens_seen": 1702821888
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923232323232323,
+      "loss": 1.4659,
+      "theoretical_loss": 3.47744298148607,
+      "tokens_seen": 1702887424
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004923030303030303,
+      "loss": 1.4347,
+      "theoretical_loss": 3.4774314672808098,
+      "tokens_seen": 1702952960
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922828282828283,
+      "loss": 1.4188,
+      "theoretical_loss": 3.477419953642716,
+      "tokens_seen": 1703018496
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922626262626262,
+      "loss": 1.4657,
+      "theoretical_loss": 3.477408440571739,
+      "tokens_seen": 1703084032
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922424242424242,
+      "loss": 1.2297,
+      "theoretical_loss": 3.4773969280678294,
+      "tokens_seen": 1703149568
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922222222222222,
+      "loss": 1.2242,
+      "theoretical_loss": 3.4773854161309363,
+      "tokens_seen": 1703215104
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004922020202020202,
+      "loss": 1.3182,
+      "theoretical_loss": 3.477373904761011,
+      "tokens_seen": 1703280640
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921818181818182,
+      "loss": 1.4721,
+      "theoretical_loss": 3.4773623939580034,
+      "tokens_seen": 1703346176
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921616161616162,
+      "loss": 1.7844,
+      "theoretical_loss": 3.477350883721863,
+      "tokens_seen": 1703411712
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921414141414142,
+      "loss": 1.3457,
+      "theoretical_loss": 3.477339374052542,
+      "tokens_seen": 1703477248
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921212121212122,
+      "loss": 1.3502,
+      "theoretical_loss": 3.477327864949989,
+      "tokens_seen": 1703542784
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004921010101010101,
+      "loss": 1.6302,
+      "theoretical_loss": 3.4773163564141547,
+      "tokens_seen": 1703608320
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920808080808081,
+      "loss": 1.3801,
+      "theoretical_loss": 3.4773048484449895,
+      "tokens_seen": 1703673856
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920606060606061,
+      "loss": 1.4358,
+      "theoretical_loss": 3.4772933410424436,
+      "tokens_seen": 1703739392
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004920404040404041,
+      "loss": 1.7846,
+      "theoretical_loss": 3.477281834206468,
+      "tokens_seen": 1703804928
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000492020202020202,
+      "loss": 1.4055,
+      "theoretical_loss": 3.4772703279370116,
+      "tokens_seen": 1703870464
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000492,
+      "loss": 1.5334,
+      "theoretical_loss": 3.477258822234026,
+      "tokens_seen": 1703936000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004919797979797979,
+      "loss": 1.5056,
+      "theoretical_loss": 3.4772473170974614,
+      "tokens_seen": 1704001536
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491959595959596,
+      "loss": 1.1744,
+      "theoretical_loss": 3.477235812527267,
+      "tokens_seen": 1704067072
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491939393939394,
+      "loss": 1.5387,
+      "theoretical_loss": 3.4772243085233945,
+      "tokens_seen": 1704132608
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/aligned_token_count": 0.3712732195854187,
+      "objective/train/avg_token_score": 0.018682070076465607,
+      "objective/train/docs_used": 965825,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.2413780689239502,
+      "objective/train/original_loss": 4.305376052856445,
+      "objective/train/theoretical_loss": 3.4772128050857933,
+      "objective/train/tokens_used": 63057376,
+      "objective/train/unlikelihood_loss": 0.054906729608774185,
+      "theoretical_loss": 3.4772128050857933,
+      "tokens_seen": 1704198144
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491919191919192,
+      "loss": 1.4768,
+      "theoretical_loss": 3.4772128050857933,
+      "tokens_seen": 1704198144
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918989898989899,
+      "loss": 1.5636,
+      "theoretical_loss": 3.4772013022144144,
+      "tokens_seen": 1704263680
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918787878787879,
+      "loss": 1.3053,
+      "theoretical_loss": 3.4771897999092083,
+      "tokens_seen": 1704329216
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918585858585859,
+      "loss": 1.5969,
+      "theoretical_loss": 3.4771782981701245,
+      "tokens_seen": 1704394752
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918383838383839,
+      "loss": 1.5039,
+      "theoretical_loss": 3.4771667969971136,
+      "tokens_seen": 1704460288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004918181818181818,
+      "loss": 1.2291,
+      "theoretical_loss": 3.4771552963901264,
+      "tokens_seen": 1704525824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917979797979798,
+      "loss": 1.3308,
+      "theoretical_loss": 3.477143796349113,
+      "tokens_seen": 1704591360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917777777777777,
+      "loss": 1.4503,
+      "theoretical_loss": 3.477132296874024,
+      "tokens_seen": 1704656896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917575757575758,
+      "loss": 1.4605,
+      "theoretical_loss": 3.47712079796481,
+      "tokens_seen": 1704722432
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917373737373737,
+      "loss": 1.553,
+      "theoretical_loss": 3.4771092996214206,
+      "tokens_seen": 1704787968
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004917171717171718,
+      "loss": 1.5346,
+      "theoretical_loss": 3.4770978018438066,
+      "tokens_seen": 1704853504
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916969696969697,
+      "loss": 1.5756,
+      "theoretical_loss": 3.4770863046319187,
+      "tokens_seen": 1704919040
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916767676767677,
+      "loss": 1.4356,
+      "theoretical_loss": 3.4770748079857072,
+      "tokens_seen": 1704984576
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916565656565657,
+      "loss": 1.441,
+      "theoretical_loss": 3.4770633119051224,
+      "tokens_seen": 1705050112
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916363636363637,
+      "loss": 1.3771,
+      "theoretical_loss": 3.4770518163901145,
+      "tokens_seen": 1705115648
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004916161616161616,
+      "loss": 1.4827,
+      "theoretical_loss": 3.4770403214406347,
+      "tokens_seen": 1705181184
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915959595959596,
+      "loss": 1.2737,
+      "theoretical_loss": 3.4770288270566327,
+      "tokens_seen": 1705246720
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915757575757576,
+      "loss": 1.6622,
+      "theoretical_loss": 3.477017333238059,
+      "tokens_seen": 1705312256
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915555555555556,
+      "loss": 1.3899,
+      "theoretical_loss": 3.4770058399848645,
+      "tokens_seen": 1705377792
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915353535353535,
+      "loss": 1.4385,
+      "theoretical_loss": 3.476994347296999,
+      "tokens_seen": 1705443328
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004915151515151515,
+      "loss": 1.3491,
+      "theoretical_loss": 3.476982855174414,
+      "tokens_seen": 1705508864
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914949494949494,
+      "loss": 1.4771,
+      "theoretical_loss": 3.476971363617059,
+      "tokens_seen": 1705574400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914747474747475,
+      "loss": 1.3391,
+      "theoretical_loss": 3.476959872624885,
+      "tokens_seen": 1705639936
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914545454545455,
+      "loss": 1.4106,
+      "theoretical_loss": 3.4769483821978424,
+      "tokens_seen": 1705705472
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914343434343435,
+      "loss": 1.2723,
+      "theoretical_loss": 3.4769368923358814,
+      "tokens_seen": 1705771008
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/aligned_token_count": 0.470796674489975,
+      "objective/train/avg_token_score": 0.008710848167538643,
+      "objective/train/docs_used": 966739,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.535874843597412,
+      "objective/train/original_loss": 3.7179017066955566,
+      "objective/train/theoretical_loss": 3.476925403038953,
+      "objective/train/tokens_used": 64695776,
+      "objective/train/unlikelihood_loss": 0.08725922554731369,
+      "theoretical_loss": 3.476925403038953,
+      "tokens_seen": 1705836544
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004914141414141414,
+      "loss": 1.7468,
+      "theoretical_loss": 3.476925403038953,
+      "tokens_seen": 1705836544
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913939393939394,
+      "loss": 1.2985,
+      "theoretical_loss": 3.4769139143070076,
+      "tokens_seen": 1705902080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913737373737374,
+      "loss": 1.3906,
+      "theoretical_loss": 3.476902426139995,
+      "tokens_seen": 1705967616
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913535353535354,
+      "loss": 1.5775,
+      "theoretical_loss": 3.4768909385378666,
+      "tokens_seen": 1706033152
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913333333333333,
+      "loss": 1.3863,
+      "theoretical_loss": 3.4768794515005723,
+      "tokens_seen": 1706098688
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004913131313131313,
+      "loss": 1.3093,
+      "theoretical_loss": 3.476867965028063,
+      "tokens_seen": 1706164224
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912929292929293,
+      "loss": 1.4804,
+      "theoretical_loss": 3.47685647912029,
+      "tokens_seen": 1706229760
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912727272727272,
+      "loss": 1.4999,
+      "theoretical_loss": 3.476844993777202,
+      "tokens_seen": 1706295296
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912525252525252,
+      "loss": 1.4509,
+      "theoretical_loss": 3.476833508998751,
+      "tokens_seen": 1706360832
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912323232323233,
+      "loss": 1.3798,
+      "theoretical_loss": 3.4768220247848873,
+      "tokens_seen": 1706426368
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004912121212121213,
+      "loss": 1.6698,
+      "theoretical_loss": 3.4768105411355608,
+      "tokens_seen": 1706491904
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911919191919192,
+      "loss": 1.5185,
+      "theoretical_loss": 3.476799058050723,
+      "tokens_seen": 1706557440
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911717171717172,
+      "loss": 1.1568,
+      "theoretical_loss": 3.4767875755303237,
+      "tokens_seen": 1706622976
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911515151515152,
+      "loss": 1.0498,
+      "theoretical_loss": 3.476776093574314,
+      "tokens_seen": 1706688512
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911313131313131,
+      "loss": 1.3549,
+      "theoretical_loss": 3.4767646121826443,
+      "tokens_seen": 1706754048
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004911111111111111,
+      "loss": 1.4073,
+      "theoretical_loss": 3.4767531313552658,
+      "tokens_seen": 1706819584
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910909090909091,
+      "loss": 1.3959,
+      "theoretical_loss": 3.4767416510921274,
+      "tokens_seen": 1706885120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491070707070707,
+      "loss": 1.2292,
+      "theoretical_loss": 3.4767301713931813,
+      "tokens_seen": 1706950656
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491050505050505,
+      "loss": 1.4875,
+      "theoretical_loss": 3.4767186922583777,
+      "tokens_seen": 1707016192
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000491030303030303,
+      "loss": 1.3406,
+      "theoretical_loss": 3.4767072136876673,
+      "tokens_seen": 1707081728
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004910101010101011,
+      "loss": 1.3963,
+      "theoretical_loss": 3.4766957356810004,
+      "tokens_seen": 1707147264
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490989898989899,
+      "loss": 1.3853,
+      "theoretical_loss": 3.4766842582383277,
+      "tokens_seen": 1707212800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490969696969697,
+      "loss": 1.2996,
+      "theoretical_loss": 3.4766727813596,
+      "tokens_seen": 1707278336
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490949494949495,
+      "loss": 1.6414,
+      "theoretical_loss": 3.476661305044768,
+      "tokens_seen": 1707343872
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490929292929293,
+      "loss": 1.3811,
+      "theoretical_loss": 3.476649829293782,
+      "tokens_seen": 1707409408
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/aligned_token_count": 0.4374389052391052,
+      "objective/train/avg_token_score": 0.005900775548070669,
+      "objective/train/docs_used": 967356,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.5339224338531494,
+      "objective/train/original_loss": 4.014285087585449,
+      "objective/train/theoretical_loss": 3.4766383541065933,
+      "objective/train/tokens_used": 66334176,
+      "objective/train/unlikelihood_loss": 0.08221994340419769,
+      "theoretical_loss": 3.4766383541065933,
+      "tokens_seen": 1707474944
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004909090909090909,
+      "loss": 1.4133,
+      "theoretical_loss": 3.4766383541065933,
+      "tokens_seen": 1707474944
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908888888888889,
+      "loss": 1.2069,
+      "theoretical_loss": 3.4766268794831516,
+      "tokens_seen": 1707540480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908686868686868,
+      "loss": 1.2704,
+      "theoretical_loss": 3.4766154054234084,
+      "tokens_seen": 1707606016
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908484848484849,
+      "loss": 1.5368,
+      "theoretical_loss": 3.4766039319273143,
+      "tokens_seen": 1707671552
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908282828282828,
+      "loss": 1.2068,
+      "theoretical_loss": 3.4765924589948196,
+      "tokens_seen": 1707737088
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004908080808080808,
+      "loss": 1.3801,
+      "theoretical_loss": 3.4765809866258754,
+      "tokens_seen": 1707802624
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907878787878787,
+      "loss": 1.5021,
+      "theoretical_loss": 3.476569514820432,
+      "tokens_seen": 1707868160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907676767676768,
+      "loss": 1.599,
+      "theoretical_loss": 3.4765580435784402,
+      "tokens_seen": 1707933696
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907474747474748,
+      "loss": 1.6612,
+      "theoretical_loss": 3.4765465728998506,
+      "tokens_seen": 1707999232
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907272727272728,
+      "loss": 1.7327,
+      "theoretical_loss": 3.476535102784614,
+      "tokens_seen": 1708064768
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004907070707070707,
+      "loss": 1.3258,
+      "theoretical_loss": 3.4765236332326817,
+      "tokens_seen": 1708130304
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906868686868687,
+      "loss": 1.2935,
+      "theoretical_loss": 3.4765121642440038,
+      "tokens_seen": 1708195840
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906666666666667,
+      "loss": 1.477,
+      "theoretical_loss": 3.476500695818531,
+      "tokens_seen": 1708261376
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906464646464647,
+      "loss": 1.5143,
+      "theoretical_loss": 3.476489227956214,
+      "tokens_seen": 1708326912
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906262626262626,
+      "loss": 1.5016,
+      "theoretical_loss": 3.476477760657004,
+      "tokens_seen": 1708392448
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004906060606060606,
+      "loss": 1.3412,
+      "theoretical_loss": 3.4764662939208515,
+      "tokens_seen": 1708457984
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905858585858585,
+      "loss": 1.3278,
+      "theoretical_loss": 3.476454827747707,
+      "tokens_seen": 1708523520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905656565656565,
+      "loss": 1.4474,
+      "theoretical_loss": 3.476443362137522,
+      "tokens_seen": 1708589056
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905454545454546,
+      "loss": 1.3127,
+      "theoretical_loss": 3.4764318970902464,
+      "tokens_seen": 1708654592
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905252525252526,
+      "loss": 1.5852,
+      "theoretical_loss": 3.4764204326058317,
+      "tokens_seen": 1708720128
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004905050505050505,
+      "loss": 1.328,
+      "theoretical_loss": 3.476408968684228,
+      "tokens_seen": 1708785664
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904848484848485,
+      "loss": 1.5521,
+      "theoretical_loss": 3.4763975053253864,
+      "tokens_seen": 1708851200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904646464646465,
+      "loss": 1.4926,
+      "theoretical_loss": 3.4763860425292576,
+      "tokens_seen": 1708916736
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904444444444445,
+      "loss": 1.3039,
+      "theoretical_loss": 3.4763745802957926,
+      "tokens_seen": 1708982272
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904242424242424,
+      "loss": 1.5436,
+      "theoretical_loss": 3.476363118624943,
+      "tokens_seen": 1709047808
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/aligned_token_count": 0.4849095940589905,
+      "objective/train/avg_token_score": 0.0028560131322592497,
+      "objective/train/docs_used": 968332,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.6418017148971558,
+      "objective/train/original_loss": 3.7804646492004395,
+      "objective/train/theoretical_loss": 3.4763516575166573,
+      "objective/train/tokens_used": 67972576,
+      "objective/train/unlikelihood_loss": 0.1123645082116127,
+      "theoretical_loss": 3.4763516575166573,
+      "tokens_seen": 1709113344
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004904040404040404,
+      "loss": 1.516,
+      "theoretical_loss": 3.4763516575166573,
+      "tokens_seen": 1709113344
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903838383838384,
+      "loss": 1.3689,
+      "theoretical_loss": 3.476340196970889,
+      "tokens_seen": 1709178880
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903636363636364,
+      "loss": 1.6257,
+      "theoretical_loss": 3.476328736987587,
+      "tokens_seen": 1709244416
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903434343434343,
+      "loss": 1.3624,
+      "theoretical_loss": 3.476317277566703,
+      "tokens_seen": 1709309952
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903232323232323,
+      "loss": 1.4273,
+      "theoretical_loss": 3.4763058187081874,
+      "tokens_seen": 1709375488
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004903030303030304,
+      "loss": 1.3265,
+      "theoretical_loss": 3.476294360411991,
+      "tokens_seen": 1709441024
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902828282828283,
+      "loss": 1.6407,
+      "theoretical_loss": 3.4762829026780655,
+      "tokens_seen": 1709506560
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902626262626263,
+      "loss": 1.5641,
+      "theoretical_loss": 3.4762714455063612,
+      "tokens_seen": 1709572096
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902424242424243,
+      "loss": 1.4669,
+      "theoretical_loss": 3.4762599888968286,
+      "tokens_seen": 1709637632
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902222222222222,
+      "loss": 1.4743,
+      "theoretical_loss": 3.476248532849419,
+      "tokens_seen": 1709703168
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004902020202020202,
+      "loss": 1.1262,
+      "theoretical_loss": 3.4762370773640834,
+      "tokens_seen": 1709768704
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901818181818182,
+      "loss": 1.4312,
+      "theoretical_loss": 3.476225622440772,
+      "tokens_seen": 1709834240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901616161616162,
+      "loss": 1.3713,
+      "theoretical_loss": 3.4762141680794367,
+      "tokens_seen": 1709899776
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901414141414141,
+      "loss": 1.4197,
+      "theoretical_loss": 3.476202714280028,
+      "tokens_seen": 1709965312
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901212121212121,
+      "loss": 1.3616,
+      "theoretical_loss": 3.476191261042496,
+      "tokens_seen": 1710030848
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004901010101010101,
+      "loss": 1.3839,
+      "theoretical_loss": 3.4761798083667923,
+      "tokens_seen": 1710096384
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000490080808080808,
+      "loss": 1.6839,
+      "theoretical_loss": 3.476168356252868,
+      "tokens_seen": 1710161920
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900606060606061,
+      "loss": 1.3635,
+      "theoretical_loss": 3.476156904700674,
+      "tokens_seen": 1710227456
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900404040404041,
+      "loss": 1.6225,
+      "theoretical_loss": 3.476145453710161,
+      "tokens_seen": 1710292992
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004900202020202021,
+      "loss": 1.3887,
+      "theoretical_loss": 3.4761340032812793,
+      "tokens_seen": 1710358528
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00049,
+      "loss": 1.4664,
+      "theoretical_loss": 3.4761225534139806,
+      "tokens_seen": 1710424064
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000489979797979798,
+      "loss": 1.388,
+      "theoretical_loss": 3.476111104108216,
+      "tokens_seen": 1710489600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000489959595959596,
+      "loss": 1.4894,
+      "theoretical_loss": 3.476099655363936,
+      "tokens_seen": 1710555136
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004899393939393939,
+      "loss": 1.2687,
+      "theoretical_loss": 3.476088207181092,
+      "tokens_seen": 1710620672
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004899191919191919,
+      "loss": 1.298,
+      "theoretical_loss": 3.4760767595596347,
+      "tokens_seen": 1710686208
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/aligned_token_count": 0.5306084752082825,
+      "objective/train/avg_token_score": 0.01167221274226904,
+      "objective/train/docs_used": 968886,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.6835694313049316,
+      "objective/train/original_loss": 4.055310249328613,
+      "objective/train/theoretical_loss": 3.4760653124995144,
+      "objective/train/tokens_used": 69610976,
+      "objective/train/unlikelihood_loss": 0.05426410958170891,
+      "theoretical_loss": 3.4760653124995144,
+      "tokens_seen": 1710751744
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898989898989899,
+      "loss": 1.7209,
+      "theoretical_loss": 3.4760653124995144,
+      "tokens_seen": 1710751744
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898787878787878,
+      "loss": 1.2871,
+      "theoretical_loss": 3.4760538660006834,
+      "tokens_seen": 1710817280
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898585858585858,
+      "loss": 1.4492,
+      "theoretical_loss": 3.4760424200630915,
+      "tokens_seen": 1710882816
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898383838383839,
+      "loss": 1.4377,
+      "theoretical_loss": 3.4760309746866906,
+      "tokens_seen": 1710948352
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004898181818181819,
+      "loss": 1.4232,
+      "theoretical_loss": 3.476019529871431,
+      "tokens_seen": 1711013888
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897979797979798,
+      "loss": 1.6333,
+      "theoretical_loss": 3.4760080856172637,
+      "tokens_seen": 1711079424
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897777777777778,
+      "loss": 1.4597,
+      "theoretical_loss": 3.475996641924141,
+      "tokens_seen": 1711144960
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897575757575758,
+      "loss": 1.5103,
+      "theoretical_loss": 3.475985198792012,
+      "tokens_seen": 1711210496
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897373737373738,
+      "loss": 1.3766,
+      "theoretical_loss": 3.4759737562208284,
+      "tokens_seen": 1711276032
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004897171717171717,
+      "loss": 1.532,
+      "theoretical_loss": 3.4759623142105416,
+      "tokens_seen": 1711341568
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896969696969697,
+      "loss": 1.4063,
+      "theoretical_loss": 3.4759508727611026,
+      "tokens_seen": 1711407104
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896767676767676,
+      "loss": 1.3932,
+      "theoretical_loss": 3.475939431872462,
+      "tokens_seen": 1711472640
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896565656565657,
+      "loss": 1.4003,
+      "theoretical_loss": 3.4759279915445713,
+      "tokens_seen": 1711538176
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896363636363636,
+      "loss": 1.335,
+      "theoretical_loss": 3.4759165517773813,
+      "tokens_seen": 1711603712
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004896161616161617,
+      "loss": 1.1782,
+      "theoretical_loss": 3.4759051125708433,
+      "tokens_seen": 1711669248
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895959595959596,
+      "loss": 1.2264,
+      "theoretical_loss": 3.475893673924908,
+      "tokens_seen": 1711734784
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895757575757576,
+      "loss": 1.4453,
+      "theoretical_loss": 3.475882235839527,
+      "tokens_seen": 1711800320
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895555555555556,
+      "loss": 1.4011,
+      "theoretical_loss": 3.4758707983146504,
+      "tokens_seen": 1711865856
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895353535353536,
+      "loss": 1.4299,
+      "theoretical_loss": 3.47585936135023,
+      "tokens_seen": 1711931392
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004895151515151515,
+      "loss": 1.5037,
+      "theoretical_loss": 3.475847924946217,
+      "tokens_seen": 1711996928
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894949494949495,
+      "loss": 1.5703,
+      "theoretical_loss": 3.475836489102562,
+      "tokens_seen": 1712062464
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894747474747475,
+      "loss": 1.4185,
+      "theoretical_loss": 3.4758250538192166,
+      "tokens_seen": 1712128000
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894545454545455,
+      "loss": 1.6266,
+      "theoretical_loss": 3.4758136190961313,
+      "tokens_seen": 1712193536
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894343434343434,
+      "loss": 1.6168,
+      "theoretical_loss": 3.475802184933258,
+      "tokens_seen": 1712259072
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004894141414141414,
+      "loss": 1.2173,
+      "theoretical_loss": 3.475790751330547,
+      "tokens_seen": 1712324608
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/aligned_token_count": 0.4318181872367859,
+      "objective/train/avg_token_score": 0.056178782135248184,
+      "objective/train/docs_used": 970327,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.3246653079986572,
+      "objective/train/original_loss": 3.9383597373962402,
+      "objective/train/theoretical_loss": 3.47577931828795,
+      "objective/train/tokens_used": 71249376,
+      "objective/train/unlikelihood_loss": 0.07031881809234619,
+      "theoretical_loss": 3.47577931828795,
+      "tokens_seen": 1712390144
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893939393939393,
+      "loss": 1.4182,
+      "theoretical_loss": 3.47577931828795,
+      "tokens_seen": 1712390144
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893737373737374,
+      "loss": 1.1773,
+      "theoretical_loss": 3.4757678858054177,
+      "tokens_seen": 1712455680
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893535353535354,
+      "loss": 1.624,
+      "theoretical_loss": 3.4757564538829016,
+      "tokens_seen": 1712521216
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893333333333334,
+      "loss": 1.4873,
+      "theoretical_loss": 3.4757450225203526,
+      "tokens_seen": 1712586752
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004893131313131313,
+      "loss": 1.3469,
+      "theoretical_loss": 3.475733591717722,
+      "tokens_seen": 1712652288
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892929292929293,
+      "loss": 1.4407,
+      "theoretical_loss": 3.4757221614749607,
+      "tokens_seen": 1712717824
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892727272727273,
+      "loss": 1.4212,
+      "theoretical_loss": 3.47571073179202,
+      "tokens_seen": 1712783360
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892525252525253,
+      "loss": 1.3738,
+      "theoretical_loss": 3.4756993026688514,
+      "tokens_seen": 1712848896
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892323232323232,
+      "loss": 1.3697,
+      "theoretical_loss": 3.475687874105406,
+      "tokens_seen": 1712914432
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004892121212121212,
+      "loss": 1.5092,
+      "theoretical_loss": 3.4756764461016347,
+      "tokens_seen": 1712979968
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891919191919192,
+      "loss": 1.2167,
+      "theoretical_loss": 3.4756650186574882,
+      "tokens_seen": 1713045504
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891717171717171,
+      "loss": 1.4599,
+      "theoretical_loss": 3.475653591772918,
+      "tokens_seen": 1713111040
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891515151515151,
+      "loss": 1.61,
+      "theoretical_loss": 3.4756421654478764,
+      "tokens_seen": 1713176576
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891313131313132,
+      "loss": 1.679,
+      "theoretical_loss": 3.475630739682313,
+      "tokens_seen": 1713242112
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004891111111111112,
+      "loss": 1.3736,
+      "theoretical_loss": 3.47561931447618,
+      "tokens_seen": 1713307648
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004890909090909091,
+      "loss": 1.2454,
+      "theoretical_loss": 3.475607889829428,
+      "tokens_seen": 1713373184
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004890707070707071,
+      "loss": 1.5084,
+      "theoretical_loss": 3.475596465742009,
+      "tokens_seen": 1713438720
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004890505050505051,
+      "loss": 1.5201,
+      "theoretical_loss": 3.475585042213874,
+      "tokens_seen": 1713504256
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000489030303030303,
+      "loss": 1.6822,
+      "theoretical_loss": 3.4755736192449733,
+      "tokens_seen": 1713569792
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000489010101010101,
+      "loss": 1.4501,
+      "theoretical_loss": 3.4755621968352592,
+      "tokens_seen": 1713635328
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000488989898989899,
+      "loss": 1.4203,
+      "theoretical_loss": 3.475550774984682,
+      "tokens_seen": 1713700864
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889696969696969,
+      "loss": 1.4855,
+      "theoretical_loss": 3.4755393536931942,
+      "tokens_seen": 1713766400
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889494949494949,
+      "loss": 1.3609,
+      "theoretical_loss": 3.4755279329607465,
+      "tokens_seen": 1713831936
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004889292929292929,
+      "loss": 1.5888,
+      "theoretical_loss": 3.4755165127872893,
+      "tokens_seen": 1713897472
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000488909090909091,
+      "loss": 1.5043,
+      "theoretical_loss": 3.4755050931727753,
+      "tokens_seen": 1713963008
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/aligned_token_count": 0.5464931726455688,
+      "objective/train/avg_token_score": 0.0072470419108867645,
+      "objective/train/docs_used": 971016,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.7423732280731201,
+      "objective/train/original_loss": 3.5915262699127197,
+      "objective/train/theoretical_loss": 3.4754936741171543,
+      "objective/train/tokens_used": 72887776,
+      "objective/train/unlikelihood_loss": 0.07072310149669647,
+      "theoretical_loss": 3.4754936741171543,
+      "tokens_seen": 1714028544
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888888888888889,
+      "loss": 1.4365,
+      "theoretical_loss": 3.4754936741171543,
+      "tokens_seen": 1714028544
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888686868686869,
+      "loss": 1.434,
+      "theoretical_loss": 3.475482255620379,
+      "tokens_seen": 1714094080
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888484848484849,
+      "loss": 1.1613,
+      "theoretical_loss": 3.4754708376823995,
+      "tokens_seen": 1714159616
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888282828282829,
+      "loss": 1.2823,
+      "theoretical_loss": 3.475459420303168,
+      "tokens_seen": 1714225152
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004888080808080808,
+      "loss": 1.2084,
+      "theoretical_loss": 3.4754480034826356,
+      "tokens_seen": 1714290688
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004887878787878788,
+      "loss": 1.1509,
+      "theoretical_loss": 3.475436587220753,
+      "tokens_seen": 1714356224
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004887676767676768,
+      "loss": 1.4293,
+      "theoretical_loss": 3.4754251715174727,
+      "tokens_seen": 1714421760
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004887474747474748,
+      "loss": 1.4948,
+      "theoretical_loss": 3.4754137563727445,
+      "tokens_seen": 1714487296
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004887272727272727,
+      "loss": 1.3427,
+      "theoretical_loss": 3.4754023417865207,
+      "tokens_seen": 1714552832
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004887070707070707,
+      "loss": 1.393,
+      "theoretical_loss": 3.4753909277587525,
+      "tokens_seen": 1714618368
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886868686868686,
+      "loss": 1.4489,
+      "theoretical_loss": 3.475379514289391,
+      "tokens_seen": 1714683904
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886666666666667,
+      "loss": 1.3675,
+      "theoretical_loss": 3.475368101378388,
+      "tokens_seen": 1714749440
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886464646464647,
+      "loss": 1.2169,
+      "theoretical_loss": 3.475356689025694,
+      "tokens_seen": 1714814976
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886262626262627,
+      "loss": 1.5594,
+      "theoretical_loss": 3.4753452772312614,
+      "tokens_seen": 1714880512
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004886060606060606,
+      "loss": 1.3374,
+      "theoretical_loss": 3.475333865995041,
+      "tokens_seen": 1714946048
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885858585858586,
+      "loss": 1.3952,
+      "theoretical_loss": 3.4753224553169835,
+      "tokens_seen": 1715011584
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885656565656566,
+      "loss": 1.4714,
+      "theoretical_loss": 3.4753110451970417,
+      "tokens_seen": 1715077120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885454545454546,
+      "loss": 1.2954,
+      "theoretical_loss": 3.475299635635166,
+      "tokens_seen": 1715142656
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885252525252525,
+      "loss": 1.5126,
+      "theoretical_loss": 3.475288226631308,
+      "tokens_seen": 1715208192
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004885050505050505,
+      "loss": 1.307,
+      "theoretical_loss": 3.475276818185419,
+      "tokens_seen": 1715273728
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884848484848484,
+      "loss": 1.3257,
+      "theoretical_loss": 3.475265410297451,
+      "tokens_seen": 1715339264
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884646464646465,
+      "loss": 1.3537,
+      "theoretical_loss": 3.4752540029673544,
+      "tokens_seen": 1715404800
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884444444444445,
+      "loss": 1.3866,
+      "theoretical_loss": 3.4752425961950815,
+      "tokens_seen": 1715470336
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884242424242425,
+      "loss": 1.4019,
+      "theoretical_loss": 3.4752311899805832,
+      "tokens_seen": 1715535872
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004884040404040404,
+      "loss": 1.1464,
+      "theoretical_loss": 3.475219784323811,
+      "tokens_seen": 1715601408
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/aligned_token_count": 0.4002932608127594,
+      "objective/train/avg_token_score": 0.017858585342764854,
+      "objective/train/docs_used": 972281,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.3489974737167358,
+      "objective/train/original_loss": 4.905088424682617,
+      "objective/train/theoretical_loss": 3.475208379224716,
+      "objective/train/tokens_used": 74526176,
+      "objective/train/unlikelihood_loss": 0.05417528748512268,
+      "theoretical_loss": 3.475208379224716,
+      "tokens_seen": 1715666944
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004883838383838384,
+      "loss": 1.526,
+      "theoretical_loss": 3.475208379224716,
+      "tokens_seen": 1715666944
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004883636363636364,
+      "loss": 1.2685,
+      "theoretical_loss": 3.475196974683251,
+      "tokens_seen": 1715732480
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004883434343434344,
+      "loss": 1.4254,
+      "theoretical_loss": 3.4751855706993657,
+      "tokens_seen": 1715798016
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004883232323232323,
+      "loss": 1.3064,
+      "theoretical_loss": 3.4751741672730123,
+      "tokens_seen": 1715863552
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004883030303030303,
+      "loss": 1.3617,
+      "theoretical_loss": 3.4751627644041427,
+      "tokens_seen": 1715929088
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004882828282828283,
+      "loss": 1.4178,
+      "theoretical_loss": 3.4751513620927073,
+      "tokens_seen": 1715994624
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004882626262626263,
+      "loss": 1.6226,
+      "theoretical_loss": 3.475139960338659,
+      "tokens_seen": 1716060160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004882424242424243,
+      "loss": 1.4401,
+      "theoretical_loss": 3.4751285591419476,
+      "tokens_seen": 1716125696
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048822222222222225,
+      "loss": 1.3443,
+      "theoretical_loss": 3.475117158502526,
+      "tokens_seen": 1716191232
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004882020202020202,
+      "loss": 1.4155,
+      "theoretical_loss": 3.4751057584203444,
+      "tokens_seen": 1716256768
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048818181818181816,
+      "loss": 1.2397,
+      "theoretical_loss": 3.4750943588953556,
+      "tokens_seen": 1716322304
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048816161616161617,
+      "loss": 1.2896,
+      "theoretical_loss": 3.47508295992751,
+      "tokens_seen": 1716387840
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004881414141414142,
+      "loss": 1.4234,
+      "theoretical_loss": 3.47507156151676,
+      "tokens_seen": 1716453376
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048812121212121214,
+      "loss": 1.6226,
+      "theoretical_loss": 3.4750601636630565,
+      "tokens_seen": 1716518912
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004881010101010101,
+      "loss": 1.3232,
+      "theoretical_loss": 3.475048766366351,
+      "tokens_seen": 1716584448
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048808080808080805,
+      "loss": 1.2095,
+      "theoretical_loss": 3.4750373696265955,
+      "tokens_seen": 1716649984
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004880606060606061,
+      "loss": 1.3368,
+      "theoretical_loss": 3.4750259734437408,
+      "tokens_seen": 1716715520
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004880404040404041,
+      "loss": 1.5763,
+      "theoretical_loss": 3.4750145778177393,
+      "tokens_seen": 1716781056
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048802020202020203,
+      "loss": 1.537,
+      "theoretical_loss": 3.4750031827485417,
+      "tokens_seen": 1716846592
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000488,
+      "loss": 1.4221,
+      "theoretical_loss": 3.4749917882360997,
+      "tokens_seen": 1716912128
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048797979797979795,
+      "loss": 1.4776,
+      "theoretical_loss": 3.4749803942803656,
+      "tokens_seen": 1716977664
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.000487959595959596,
+      "loss": 1.6911,
+      "theoretical_loss": 3.4749690008812903,
+      "tokens_seen": 1717043200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048793939393939397,
+      "loss": 1.6163,
+      "theoretical_loss": 3.4749576080388254,
+      "tokens_seen": 1717108736
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004879191919191919,
+      "loss": 1.5933,
+      "theoretical_loss": 3.4749462157529227,
+      "tokens_seen": 1717174272
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004878989898989899,
+      "loss": 1.4458,
+      "theoretical_loss": 3.4749348240235336,
+      "tokens_seen": 1717239808
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/aligned_token_count": 0.34756842255592346,
+      "objective/train/avg_token_score": 0.022121969610452652,
+      "objective/train/docs_used": 972817,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.2492351531982422,
+      "objective/train/original_loss": 4.464137554168701,
+      "objective/train/theoretical_loss": 3.47492343285061,
+      "objective/train/tokens_used": 76164576,
+      "objective/train/unlikelihood_loss": 0.06865523010492325,
+      "theoretical_loss": 3.47492343285061,
+      "tokens_seen": 1717305344
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048787878787878784,
+      "loss": 1.4663,
+      "theoretical_loss": 3.47492343285061,
+      "tokens_seen": 1717305344
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004878585858585859,
+      "loss": 1.6475,
+      "theoretical_loss": 3.474912042234102,
+      "tokens_seen": 1717370880
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048783838383838386,
+      "loss": 1.3615,
+      "theoretical_loss": 3.4749006521739636,
+      "tokens_seen": 1717436416
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004878181818181818,
+      "loss": 1.3724,
+      "theoretical_loss": 3.4748892626701444,
+      "tokens_seen": 1717501952
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004877979797979798,
+      "loss": 1.4726,
+      "theoretical_loss": 3.4748778737225976,
+      "tokens_seen": 1717567488
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004877777777777778,
+      "loss": 1.5526,
+      "theoretical_loss": 3.4748664853312734,
+      "tokens_seen": 1717633024
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004877575757575758,
+      "loss": 1.3223,
+      "theoretical_loss": 3.474855097496124,
+      "tokens_seen": 1717698560
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048773737373737375,
+      "loss": 1.6783,
+      "theoretical_loss": 3.474843710217101,
+      "tokens_seen": 1717764096
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004877171717171717,
+      "loss": 1.3255,
+      "theoretical_loss": 3.474832323494156,
+      "tokens_seen": 1717829632
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048769696969696967,
+      "loss": 1.3216,
+      "theoretical_loss": 3.474820937327241,
+      "tokens_seen": 1717895168
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004876767676767677,
+      "loss": 1.3008,
+      "theoretical_loss": 3.474809551716307,
+      "tokens_seen": 1717960704
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004876565656565657,
+      "loss": 1.1347,
+      "theoretical_loss": 3.474798166661306,
+      "tokens_seen": 1718026240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048763636363636365,
+      "loss": 1.4412,
+      "theoretical_loss": 3.4747867821621896,
+      "tokens_seen": 1718091776
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004876161616161616,
+      "loss": 1.1156,
+      "theoretical_loss": 3.4747753982189096,
+      "tokens_seen": 1718157312
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004875959595959596,
+      "loss": 1.4876,
+      "theoretical_loss": 3.474764014831417,
+      "tokens_seen": 1718222848
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048757575757575757,
+      "loss": 1.4029,
+      "theoretical_loss": 3.4747526319996647,
+      "tokens_seen": 1718288384
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004875555555555556,
+      "loss": 1.3172,
+      "theoretical_loss": 3.474741249723603,
+      "tokens_seen": 1718353920
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048753535353535354,
+      "loss": 1.2698,
+      "theoretical_loss": 3.4747298680031844,
+      "tokens_seen": 1718419456
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004875151515151515,
+      "loss": 1.5606,
+      "theoretical_loss": 3.4747184868383605,
+      "tokens_seen": 1718484992
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004874949494949495,
+      "loss": 1.4975,
+      "theoretical_loss": 3.474707106229083,
+      "tokens_seen": 1718550528
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048747474747474746,
+      "loss": 1.5241,
+      "theoretical_loss": 3.4746957261753035,
+      "tokens_seen": 1718616064
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004874545454545455,
+      "loss": 1.5173,
+      "theoretical_loss": 3.4746843466769737,
+      "tokens_seen": 1718681600
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048743434343434343,
+      "loss": 1.5222,
+      "theoretical_loss": 3.4746729677340453,
+      "tokens_seen": 1718747136
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.00048741414141414144,
+      "loss": 1.4428,
+      "theoretical_loss": 3.47466158934647,
+      "tokens_seen": 1718812672
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 0.0004873939393939394,
+      "loss": 1.2955,
+      "theoretical_loss": 3.4746502115141995,
+      "tokens_seen": 1718878208
+    },
+    {
+      "epoch": 0.03,
+      "objective/train/aligned_token_count": 0.3179985284805298,
+      "objective/train/avg_token_score": 0.02353089302778244,
+      "objective/train/docs_used": 973424,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.0046734809875488,
+      "objective/train/original_loss": 4.215363502502441,
+      "objective/train/theoretical_loss": 3.4746388342371857,
+      "objective/train/tokens_used": 77802976,
+      "objective/train/unlikelihood_loss": 0.07478584349155426,
+      "theoretical_loss": 3.4746388342371857,
+      "tokens_seen": 1718943744
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048737373737373736,
+      "loss": 1.2882,
+      "theoretical_loss": 3.4746388342371857,
+      "tokens_seen": 1718943744
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048735353535353537,
+      "loss": 1.5482,
+      "theoretical_loss": 3.47462745751538,
+      "tokens_seen": 1719009280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004873333333333333,
+      "loss": 1.1838,
+      "theoretical_loss": 3.474616081348735,
+      "tokens_seen": 1719074816
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048731313131313134,
+      "loss": 1.438,
+      "theoretical_loss": 3.4746047057372014,
+      "tokens_seen": 1719140352
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004872929292929293,
+      "loss": 1.4355,
+      "theoretical_loss": 3.4745933306807313,
+      "tokens_seen": 1719205888
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048727272727272725,
+      "loss": 1.2308,
+      "theoretical_loss": 3.474581956179277,
+      "tokens_seen": 1719271424
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048725252525252526,
+      "loss": 1.5441,
+      "theoretical_loss": 3.474570582232789,
+      "tokens_seen": 1719336960
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048723232323232327,
+      "loss": 1.4567,
+      "theoretical_loss": 3.4745592088412205,
+      "tokens_seen": 1719402496
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048721212121212123,
+      "loss": 1.5781,
+      "theoretical_loss": 3.4745478360045228,
+      "tokens_seen": 1719468032
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004871919191919192,
+      "loss": 1.4201,
+      "theoretical_loss": 3.4745364637226466,
+      "tokens_seen": 1719533568
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048717171717171714,
+      "loss": 1.5531,
+      "theoretical_loss": 3.4745250919955453,
+      "tokens_seen": 1719599104
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048715151515151515,
+      "loss": 1.0908,
+      "theoretical_loss": 3.47451372082317,
+      "tokens_seen": 1719664640
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048713131313131316,
+      "loss": 1.6216,
+      "theoretical_loss": 3.474502350205472,
+      "tokens_seen": 1719730176
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004871111111111111,
+      "loss": 1.2598,
+      "theoretical_loss": 3.474490980142404,
+      "tokens_seen": 1719795712
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004870909090909091,
+      "loss": 1.3527,
+      "theoretical_loss": 3.4744796106339173,
+      "tokens_seen": 1719861248
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048707070707070704,
+      "loss": 1.3626,
+      "theoretical_loss": 3.474468241679964,
+      "tokens_seen": 1719926784
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004870505050505051,
+      "loss": 1.4276,
+      "theoretical_loss": 3.474456873280496,
+      "tokens_seen": 1719992320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048703030303030306,
+      "loss": 1.5703,
+      "theoretical_loss": 3.4744455054354644,
+      "tokens_seen": 1720057856
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000487010101010101,
+      "loss": 1.4335,
+      "theoretical_loss": 3.474434138144822,
+      "tokens_seen": 1720123392
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048698989898989897,
+      "loss": 1.4018,
+      "theoretical_loss": 3.47442277140852,
+      "tokens_seen": 1720188928
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000486969696969697,
+      "loss": 1.4003,
+      "theoretical_loss": 3.47441140522651,
+      "tokens_seen": 1720254464
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000486949494949495,
+      "loss": 1.5975,
+      "theoretical_loss": 3.474400039598745,
+      "tokens_seen": 1720320000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048692929292929295,
+      "loss": 1.3209,
+      "theoretical_loss": 3.4743886745251755,
+      "tokens_seen": 1720385536
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004869090909090909,
+      "loss": 1.5469,
+      "theoretical_loss": 3.4743773100057544,
+      "tokens_seen": 1720451072
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048688888888888886,
+      "loss": 1.3673,
+      "theoretical_loss": 3.4743659460404333,
+      "tokens_seen": 1720516608
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/aligned_token_count": 0.5211998820304871,
+      "objective/train/avg_token_score": 0.0027845173608511686,
+      "objective/train/docs_used": 974650,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.7277867794036865,
+      "objective/train/original_loss": 3.4647514820098877,
+      "objective/train/theoretical_loss": 3.4743545826291635,
+      "objective/train/tokens_used": 79441376,
+      "objective/train/unlikelihood_loss": 0.10759128630161285,
+      "theoretical_loss": 3.4743545826291635,
+      "tokens_seen": 1720582144
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048686868686868693,
+      "loss": 1.5171,
+      "theoretical_loss": 3.4743545826291635,
+      "tokens_seen": 1720582144
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004868484848484849,
+      "loss": 1.3346,
+      "theoretical_loss": 3.4743432197718978,
+      "tokens_seen": 1720647680
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048682828282828284,
+      "loss": 1.2947,
+      "theoretical_loss": 3.4743318574685875,
+      "tokens_seen": 1720713216
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004868080808080808,
+      "loss": 1.3649,
+      "theoretical_loss": 3.4743204957191844,
+      "tokens_seen": 1720778752
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048678787878787876,
+      "loss": 1.3602,
+      "theoretical_loss": 3.474309134523641,
+      "tokens_seen": 1720844288
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004867676767676768,
+      "loss": 1.3379,
+      "theoretical_loss": 3.4742977738819087,
+      "tokens_seen": 1720909824
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004867474747474748,
+      "loss": 1.5094,
+      "theoretical_loss": 3.4742864137939398,
+      "tokens_seen": 1720975360
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048672727272727274,
+      "loss": 1.4478,
+      "theoretical_loss": 3.474275054259685,
+      "tokens_seen": 1721040896
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004867070707070707,
+      "loss": 1.4563,
+      "theoretical_loss": 3.4742636952790984,
+      "tokens_seen": 1721106432
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004866868686868687,
+      "loss": 1.5572,
+      "theoretical_loss": 3.47425233685213,
+      "tokens_seen": 1721171968
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004866666666666667,
+      "loss": 1.3569,
+      "theoretical_loss": 3.474240978978733,
+      "tokens_seen": 1721237504
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048664646464646467,
+      "loss": 1.2837,
+      "theoretical_loss": 3.474229621658859,
+      "tokens_seen": 1721303040
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048662626262626263,
+      "loss": 1.4573,
+      "theoretical_loss": 3.474218264892459,
+      "tokens_seen": 1721368576
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004866060606060606,
+      "loss": 1.4406,
+      "theoretical_loss": 3.4742069086794864,
+      "tokens_seen": 1721434112
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004865858585858586,
+      "loss": 1.2444,
+      "theoretical_loss": 3.474195553019892,
+      "tokens_seen": 1721499648
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004865656565656566,
+      "loss": 1.3948,
+      "theoretical_loss": 3.4741841979136288,
+      "tokens_seen": 1721565184
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048654545454545456,
+      "loss": 1.4,
+      "theoretical_loss": 3.4741728433606482,
+      "tokens_seen": 1721630720
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004865252525252525,
+      "loss": 1.5166,
+      "theoretical_loss": 3.4741614893609016,
+      "tokens_seen": 1721696256
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048650505050505053,
+      "loss": 1.5006,
+      "theoretical_loss": 3.4741501359143423,
+      "tokens_seen": 1721761792
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004864848484848485,
+      "loss": 1.2737,
+      "theoretical_loss": 3.4741387830209214,
+      "tokens_seen": 1721827328
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004864646464646465,
+      "loss": 1.3364,
+      "theoretical_loss": 3.474127430680591,
+      "tokens_seen": 1721892864
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048644444444444446,
+      "loss": 1.4177,
+      "theoretical_loss": 3.474116078893303,
+      "tokens_seen": 1721958400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004864242424242424,
+      "loss": 1.3975,
+      "theoretical_loss": 3.47410472765901,
+      "tokens_seen": 1722023936
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004864040404040404,
+      "loss": 1.3059,
+      "theoretical_loss": 3.474093376977664,
+      "tokens_seen": 1722089472
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004863838383838384,
+      "loss": 1.4123,
+      "theoretical_loss": 3.474082026849216,
+      "tokens_seen": 1722155008
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/aligned_token_count": 0.3375488817691803,
+      "objective/train/avg_token_score": 0.013777061365544796,
+      "objective/train/docs_used": 975130,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.2517693042755127,
+      "objective/train/original_loss": 5.1249237060546875,
+      "objective/train/theoretical_loss": 3.4740706772736187,
+      "objective/train/tokens_used": 81079776,
+      "objective/train/unlikelihood_loss": 0.040009669959545135,
+      "theoretical_loss": 3.4740706772736187,
+      "tokens_seen": 1722220544
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004863636363636364,
+      "loss": 1.5127,
+      "theoretical_loss": 3.4740706772736187,
+      "tokens_seen": 1722220544
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048634343434343435,
+      "loss": 1.2964,
+      "theoretical_loss": 3.4740593282508243,
+      "tokens_seen": 1722286080
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048632323232323236,
+      "loss": 1.4136,
+      "theoretical_loss": 3.474047979780785,
+      "tokens_seen": 1722351616
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004863030303030303,
+      "loss": 1.4117,
+      "theoretical_loss": 3.474036631863452,
+      "tokens_seen": 1722417152
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004862828282828283,
+      "loss": 1.3667,
+      "theoretical_loss": 3.4740252844987785,
+      "tokens_seen": 1722482688
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004862626262626263,
+      "loss": 1.2363,
+      "theoretical_loss": 3.474013937686715,
+      "tokens_seen": 1722548224
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048624242424242424,
+      "loss": 1.4347,
+      "theoretical_loss": 3.4740025914272152,
+      "tokens_seen": 1722613760
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048622222222222225,
+      "loss": 1.3785,
+      "theoretical_loss": 3.4739912457202307,
+      "tokens_seen": 1722679296
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004862020202020202,
+      "loss": 1.4418,
+      "theoretical_loss": 3.4739799005657126,
+      "tokens_seen": 1722744832
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048618181818181817,
+      "loss": 1.4983,
+      "theoretical_loss": 3.473968555963614,
+      "tokens_seen": 1722810368
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004861616161616162,
+      "loss": 1.4721,
+      "theoretical_loss": 3.473957211913887,
+      "tokens_seen": 1722875904
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004861414141414142,
+      "loss": 1.4795,
+      "theoretical_loss": 3.4739458684164832,
+      "tokens_seen": 1722941440
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048612121212121215,
+      "loss": 1.3607,
+      "theoretical_loss": 3.473934525471355,
+      "tokens_seen": 1723006976
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004861010101010101,
+      "loss": 1.4223,
+      "theoretical_loss": 3.4739231830784543,
+      "tokens_seen": 1723072512
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048608080808080806,
+      "loss": 1.5619,
+      "theoretical_loss": 3.473911841237733,
+      "tokens_seen": 1723138048
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048606060606060607,
+      "loss": 1.4897,
+      "theoretical_loss": 3.473900499949144,
+      "tokens_seen": 1723203584
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004860404040404041,
+      "loss": 1.3993,
+      "theoretical_loss": 3.4738891592126393,
+      "tokens_seen": 1723269120
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048602020202020204,
+      "loss": 1.408,
+      "theoretical_loss": 3.4738778190281705,
+      "tokens_seen": 1723334656
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000486,
+      "loss": 1.3879,
+      "theoretical_loss": 3.4738664793956895,
+      "tokens_seen": 1723400192
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048597979797979795,
+      "loss": 1.4359,
+      "theoretical_loss": 3.473855140315149,
+      "tokens_seen": 1723465728
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048595959595959596,
+      "loss": 1.4494,
+      "theoretical_loss": 3.473843801786501,
+      "tokens_seen": 1723531264
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000485939393939394,
+      "loss": 1.3349,
+      "theoretical_loss": 3.473832463809698,
+      "tokens_seen": 1723596800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048591919191919193,
+      "loss": 1.4685,
+      "theoretical_loss": 3.4738211263846916,
+      "tokens_seen": 1723662336
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004858989898989899,
+      "loss": 1.3586,
+      "theoretical_loss": 3.473809789511434,
+      "tokens_seen": 1723727872
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048587878787878784,
+      "loss": 1.3079,
+      "theoretical_loss": 3.473798453189878,
+      "tokens_seen": 1723793408
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/aligned_token_count": 0.4409824013710022,
+      "objective/train/avg_token_score": 0.026003144681453705,
+      "objective/train/docs_used": 976352,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.4241589307785034,
+      "objective/train/original_loss": 4.103183746337891,
+      "objective/train/theoretical_loss": 3.4737871174199757,
+      "objective/train/tokens_used": 82718176,
+      "objective/train/unlikelihood_loss": 0.09437726438045502,
+      "theoretical_loss": 3.4737871174199757,
+      "tokens_seen": 1723858944
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004858585858585859,
+      "loss": 1.4386,
+      "theoretical_loss": 3.4737871174199757,
+      "tokens_seen": 1723858944
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048583838383838387,
+      "loss": 1.3626,
+      "theoretical_loss": 3.4737757822016784,
+      "tokens_seen": 1723924480
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004858181818181818,
+      "loss": 1.4696,
+      "theoretical_loss": 3.473764447534939,
+      "tokens_seen": 1723990016
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004857979797979798,
+      "loss": 1.3754,
+      "theoretical_loss": 3.4737531134197095,
+      "tokens_seen": 1724055552
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048577777777777774,
+      "loss": 1.4346,
+      "theoretical_loss": 3.473741779855942,
+      "tokens_seen": 1724121088
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004857575757575758,
+      "loss": 1.4929,
+      "theoretical_loss": 3.4737304468435894,
+      "tokens_seen": 1724186624
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048573737373737376,
+      "loss": 1.4366,
+      "theoretical_loss": 3.473719114382603,
+      "tokens_seen": 1724252160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004857171717171717,
+      "loss": 1.2766,
+      "theoretical_loss": 3.4737077824729354,
+      "tokens_seen": 1724317696
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004856969696969697,
+      "loss": 1.3714,
+      "theoretical_loss": 3.4736964511145385,
+      "tokens_seen": 1724383232
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004856767676767677,
+      "loss": 1.2763,
+      "theoretical_loss": 3.473685120307365,
+      "tokens_seen": 1724448768
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004856565656565657,
+      "loss": 1.4836,
+      "theoretical_loss": 3.4736737900513672,
+      "tokens_seen": 1724514304
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048563636363636365,
+      "loss": 1.4698,
+      "theoretical_loss": 3.473662460346497,
+      "tokens_seen": 1724579840
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004856161616161616,
+      "loss": 1.4449,
+      "theoretical_loss": 3.4736511311927067,
+      "tokens_seen": 1724645376
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048559595959595957,
+      "loss": 1.3052,
+      "theoretical_loss": 3.4736398025899486,
+      "tokens_seen": 1724710912
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004855757575757576,
+      "loss": 1.565,
+      "theoretical_loss": 3.4736284745381756,
+      "tokens_seen": 1724776448
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004855555555555556,
+      "loss": 1.3919,
+      "theoretical_loss": 3.4736171470373387,
+      "tokens_seen": 1724841984
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048553535353535355,
+      "loss": 1.5023,
+      "theoretical_loss": 3.473605820087391,
+      "tokens_seen": 1724907520
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004855151515151515,
+      "loss": 1.2998,
+      "theoretical_loss": 3.4735944936882848,
+      "tokens_seen": 1724973056
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004854949494949495,
+      "loss": 1.6207,
+      "theoretical_loss": 3.473583167839972,
+      "tokens_seen": 1725038592
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048547474747474747,
+      "loss": 1.2696,
+      "theoretical_loss": 3.473571842542405,
+      "tokens_seen": 1725104128
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004854545454545455,
+      "loss": 1.6765,
+      "theoretical_loss": 3.473560517795536,
+      "tokens_seen": 1725169664
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048543434343434344,
+      "loss": 1.5439,
+      "theoretical_loss": 3.473549193599318,
+      "tokens_seen": 1725235200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004854141414141414,
+      "loss": 1.5656,
+      "theoretical_loss": 3.4735378699537027,
+      "tokens_seen": 1725300736
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004853939393939394,
+      "loss": 1.2932,
+      "theoretical_loss": 3.4735265468586425,
+      "tokens_seen": 1725366272
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048537373737373736,
+      "loss": 1.4775,
+      "theoretical_loss": 3.4735152243140894,
+      "tokens_seen": 1725431808
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/aligned_token_count": 0.4415322542190552,
+      "objective/train/avg_token_score": 0.00970387365669012,
+      "objective/train/docs_used": 977030,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.4506078958511353,
+      "objective/train/original_loss": 4.433774948120117,
+      "objective/train/theoretical_loss": 3.4735039023199965,
+      "objective/train/tokens_used": 84356576,
+      "objective/train/unlikelihood_loss": 0.02620289847254753,
+      "theoretical_loss": 3.4735039023199965,
+      "tokens_seen": 1725497344
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004853535353535354,
+      "loss": 1.7168,
+      "theoretical_loss": 3.4735039023199965,
+      "tokens_seen": 1725497344
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048533333333333333,
+      "loss": 1.2021,
+      "theoretical_loss": 3.4734925808763153,
+      "tokens_seen": 1725562880
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048531313131313134,
+      "loss": 1.4966,
+      "theoretical_loss": 3.473481259982999,
+      "tokens_seen": 1725628416
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004852929292929293,
+      "loss": 1.4124,
+      "theoretical_loss": 3.4734699396399993,
+      "tokens_seen": 1725693952
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004852727272727273,
+      "loss": 1.3887,
+      "theoretical_loss": 3.4734586198472686,
+      "tokens_seen": 1725759488
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048525252525252527,
+      "loss": 1.1216,
+      "theoretical_loss": 3.4734473006047595,
+      "tokens_seen": 1725825024
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004852323232323232,
+      "loss": 1.3029,
+      "theoretical_loss": 3.4734359819124245,
+      "tokens_seen": 1725890560
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048521212121212123,
+      "loss": 1.3902,
+      "theoretical_loss": 3.4734246637702153,
+      "tokens_seen": 1725956096
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004851919191919192,
+      "loss": 1.34,
+      "theoretical_loss": 3.4734133461780847,
+      "tokens_seen": 1726021632
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004851717171717172,
+      "loss": 1.4358,
+      "theoretical_loss": 3.4734020291359853,
+      "tokens_seen": 1726087168
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048515151515151516,
+      "loss": 1.4642,
+      "theoretical_loss": 3.473390712643869,
+      "tokens_seen": 1726152704
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048513131313131317,
+      "loss": 1.3417,
+      "theoretical_loss": 3.473379396701689,
+      "tokens_seen": 1726218240
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004851111111111111,
+      "loss": 1.1703,
+      "theoretical_loss": 3.473368081309397,
+      "tokens_seen": 1726283776
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004850909090909091,
+      "loss": 1.6721,
+      "theoretical_loss": 3.473356766466945,
+      "tokens_seen": 1726349312
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004850707070707071,
+      "loss": 1.3141,
+      "theoretical_loss": 3.4733454521742866,
+      "tokens_seen": 1726414848
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048505050505050505,
+      "loss": 1.2502,
+      "theoretical_loss": 3.4733341384313734,
+      "tokens_seen": 1726480384
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048503030303030306,
+      "loss": 1.2986,
+      "theoretical_loss": 3.4733228252381574,
+      "tokens_seen": 1726545920
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000485010101010101,
+      "loss": 1.3311,
+      "theoretical_loss": 3.4733115125945924,
+      "tokens_seen": 1726611456
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000484989898989899,
+      "loss": 1.5198,
+      "theoretical_loss": 3.47330020050063,
+      "tokens_seen": 1726676992
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000484969696969697,
+      "loss": 1.2449,
+      "theoretical_loss": 3.4732888889562226,
+      "tokens_seen": 1726742528
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000484949494949495,
+      "loss": 1.345,
+      "theoretical_loss": 3.4732775779613227,
+      "tokens_seen": 1726808064
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048492929292929296,
+      "loss": 1.3956,
+      "theoretical_loss": 3.473266267515883,
+      "tokens_seen": 1726873600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004849090909090909,
+      "loss": 1.3801,
+      "theoretical_loss": 3.4732549576198553,
+      "tokens_seen": 1726939136
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048488888888888887,
+      "loss": 1.3834,
+      "theoretical_loss": 3.4732436482731925,
+      "tokens_seen": 1727004672
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004848686868686869,
+      "loss": 1.4937,
+      "theoretical_loss": 3.4732323394758478,
+      "tokens_seen": 1727070208
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/aligned_token_count": 0.39345064759254456,
+      "objective/train/avg_token_score": 0.009139729663729668,
+      "objective/train/docs_used": 978106,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.4235575199127197,
+      "objective/train/original_loss": 4.360210418701172,
+      "objective/train/theoretical_loss": 3.473221031227773,
+      "objective/train/tokens_used": 85994976,
+      "objective/train/unlikelihood_loss": 0.08843415230512619,
+      "theoretical_loss": 3.473221031227773,
+      "tokens_seen": 1727135744
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004848484848484849,
+      "loss": 1.5847,
+      "theoretical_loss": 3.473221031227773,
+      "tokens_seen": 1727135744
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048482828282828285,
+      "loss": 1.4794,
+      "theoretical_loss": 3.47320972352892,
+      "tokens_seen": 1727201280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004848080808080808,
+      "loss": 1.4167,
+      "theoretical_loss": 3.473198416379242,
+      "tokens_seen": 1727266816
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048478787878787876,
+      "loss": 1.2351,
+      "theoretical_loss": 3.4731871097786913,
+      "tokens_seen": 1727332352
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048476767676767683,
+      "loss": 1.3563,
+      "theoretical_loss": 3.4731758037272202,
+      "tokens_seen": 1727397888
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004847474747474748,
+      "loss": 1.291,
+      "theoretical_loss": 3.473164498224782,
+      "tokens_seen": 1727463424
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048472727272727274,
+      "loss": 1.4047,
+      "theoretical_loss": 3.4731531932713287,
+      "tokens_seen": 1727528960
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004847070707070707,
+      "loss": 1.3456,
+      "theoretical_loss": 3.473141888866812,
+      "tokens_seen": 1727594496
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048468686868686865,
+      "loss": 1.3892,
+      "theoretical_loss": 3.4731305850111855,
+      "tokens_seen": 1727660032
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004846666666666667,
+      "loss": 1.3859,
+      "theoretical_loss": 3.473119281704402,
+      "tokens_seen": 1727725568
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004846464646464647,
+      "loss": 1.1975,
+      "theoretical_loss": 3.473107978946413,
+      "tokens_seen": 1727791104
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048462626262626263,
+      "loss": 1.3792,
+      "theoretical_loss": 3.4730966767371716,
+      "tokens_seen": 1727856640
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004846060606060606,
+      "loss": 1.3253,
+      "theoretical_loss": 3.4730853750766304,
+      "tokens_seen": 1727922176
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048458585858585855,
+      "loss": 1.5187,
+      "theoretical_loss": 3.4730740739647414,
+      "tokens_seen": 1727987712
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004845656565656566,
+      "loss": 1.4738,
+      "theoretical_loss": 3.4730627734014576,
+      "tokens_seen": 1728053248
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048454545454545457,
+      "loss": 1.1208,
+      "theoretical_loss": 3.4730514733867315,
+      "tokens_seen": 1728118784
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004845252525252525,
+      "loss": 1.3628,
+      "theoretical_loss": 3.4730401739205163,
+      "tokens_seen": 1728184320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004845050505050505,
+      "loss": 1.4873,
+      "theoretical_loss": 3.4730288750027634,
+      "tokens_seen": 1728249856
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004844848484848485,
+      "loss": 1.3379,
+      "theoretical_loss": 3.473017576633426,
+      "tokens_seen": 1728315392
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004844646464646465,
+      "loss": 1.2096,
+      "theoretical_loss": 3.4730062788124565,
+      "tokens_seen": 1728380928
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048444444444444446,
+      "loss": 1.2331,
+      "theoretical_loss": 3.472994981539808,
+      "tokens_seen": 1728446464
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004844242424242424,
+      "loss": 1.5172,
+      "theoretical_loss": 3.4729836848154325,
+      "tokens_seen": 1728512000
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004844040404040404,
+      "loss": 1.4394,
+      "theoretical_loss": 3.472972388639283,
+      "tokens_seen": 1728577536
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004843838383838384,
+      "loss": 1.4188,
+      "theoretical_loss": 3.4729610930113117,
+      "tokens_seen": 1728643072
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004843636363636364,
+      "loss": 1.305,
+      "theoretical_loss": 3.472949797931472,
+      "tokens_seen": 1728708608
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/aligned_token_count": 0.3660801649093628,
+      "objective/train/avg_token_score": 0.007394853513687849,
+      "objective/train/docs_used": 978835,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.256290316581726,
+      "objective/train/original_loss": 4.4458208084106445,
+      "objective/train/theoretical_loss": 3.472938503399715,
+      "objective/train/tokens_used": 87633376,
+      "objective/train/unlikelihood_loss": 0.06266473978757858,
+      "theoretical_loss": 3.472938503399715,
+      "tokens_seen": 1728774144
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048434343434343435,
+      "loss": 1.5173,
+      "theoretical_loss": 3.472938503399715,
+      "tokens_seen": 1728774144
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004843232323232323,
+      "loss": 1.6362,
+      "theoretical_loss": 3.472927209415995,
+      "tokens_seen": 1728839680
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004843030303030303,
+      "loss": 1.2885,
+      "theoretical_loss": 3.4729159159802636,
+      "tokens_seen": 1728905216
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004842828282828283,
+      "loss": 1.5108,
+      "theoretical_loss": 3.4729046230924743,
+      "tokens_seen": 1728970752
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004842626262626263,
+      "loss": 1.5734,
+      "theoretical_loss": 3.472893330752579,
+      "tokens_seen": 1729036288
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048424242424242425,
+      "loss": 1.2754,
+      "theoretical_loss": 3.4728820389605306,
+      "tokens_seen": 1729101824
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004842222222222222,
+      "loss": 1.3543,
+      "theoretical_loss": 3.4728707477162817,
+      "tokens_seen": 1729167360
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004842020202020202,
+      "loss": 1.4882,
+      "theoretical_loss": 3.472859457019785,
+      "tokens_seen": 1729232896
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048418181818181817,
+      "loss": 1.288,
+      "theoretical_loss": 3.4728481668709934,
+      "tokens_seen": 1729298432
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004841616161616162,
+      "loss": 1.1096,
+      "theoretical_loss": 3.4728368772698586,
+      "tokens_seen": 1729363968
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048414141414141414,
+      "loss": 1.3439,
+      "theoretical_loss": 3.4728255882163346,
+      "tokens_seen": 1729429504
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048412121212121215,
+      "loss": 1.3121,
+      "theoretical_loss": 3.4728142997103735,
+      "tokens_seen": 1729495040
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004841010101010101,
+      "loss": 1.3209,
+      "theoretical_loss": 3.472803011751928,
+      "tokens_seen": 1729560576
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048408080808080807,
+      "loss": 1.3692,
+      "theoretical_loss": 3.472791724340951,
+      "tokens_seen": 1729626112
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004840606060606061,
+      "loss": 1.5298,
+      "theoretical_loss": 3.472780437477395,
+      "tokens_seen": 1729691648
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048404040404040403,
+      "loss": 1.4671,
+      "theoretical_loss": 3.472769151161213,
+      "tokens_seen": 1729757184
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048402020202020204,
+      "loss": 1.5067,
+      "theoretical_loss": 3.4727578653923565,
+      "tokens_seen": 1729822720
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000484,
+      "loss": 1.5815,
+      "theoretical_loss": 3.4727465801707798,
+      "tokens_seen": 1729888256
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048397979797979796,
+      "loss": 1.5065,
+      "theoretical_loss": 3.472735295496435,
+      "tokens_seen": 1729953792
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048395959595959597,
+      "loss": 1.3621,
+      "theoretical_loss": 3.4727240113692748,
+      "tokens_seen": 1730019328
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000483939393939394,
+      "loss": 1.2581,
+      "theoretical_loss": 3.472712727789252,
+      "tokens_seen": 1730084864
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048391919191919194,
+      "loss": 1.4585,
+      "theoretical_loss": 3.4727014447563196,
+      "tokens_seen": 1730150400
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004838989898989899,
+      "loss": 1.3508,
+      "theoretical_loss": 3.472690162270429,
+      "tokens_seen": 1730215936
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048387878787878785,
+      "loss": 1.4799,
+      "theoretical_loss": 3.472678880331535,
+      "tokens_seen": 1730281472
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048385858585858586,
+      "loss": 1.3777,
+      "theoretical_loss": 3.4726675989395894,
+      "tokens_seen": 1730347008
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/aligned_token_count": 0.46120479702949524,
+      "objective/train/avg_token_score": 0.012367086485028267,
+      "objective/train/docs_used": 979527,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.5714759826660156,
+      "objective/train/original_loss": 4.267512321472168,
+      "objective/train/theoretical_loss": 3.472656318094545,
+      "objective/train/tokens_used": 89271776,
+      "objective/train/unlikelihood_loss": 0.061104901134967804,
+      "theoretical_loss": 3.472656318094545,
+      "tokens_seen": 1730412544
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048383838383838387,
+      "loss": 1.3963,
+      "theoretical_loss": 3.472656318094545,
+      "tokens_seen": 1730412544
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048381818181818183,
+      "loss": 1.4758,
+      "theoretical_loss": 3.4726450377963545,
+      "tokens_seen": 1730478080
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004837979797979798,
+      "loss": 1.2657,
+      "theoretical_loss": 3.47263375804497,
+      "tokens_seen": 1730543616
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048377777777777774,
+      "loss": 1.3777,
+      "theoretical_loss": 3.472622478840346,
+      "tokens_seen": 1730609152
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004837575757575758,
+      "loss": 1.4637,
+      "theoretical_loss": 3.472611200182434,
+      "tokens_seen": 1730674688
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048373737373737377,
+      "loss": 1.3547,
+      "theoretical_loss": 3.472599922071187,
+      "tokens_seen": 1730740224
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004837171717171717,
+      "loss": 1.4689,
+      "theoretical_loss": 3.472588644506558,
+      "tokens_seen": 1730805760
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004836969696969697,
+      "loss": 1.6301,
+      "theoretical_loss": 3.4725773674884994,
+      "tokens_seen": 1730871296
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048367676767676764,
+      "loss": 1.5216,
+      "theoretical_loss": 3.4725660910169647,
+      "tokens_seen": 1730936832
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004836565656565657,
+      "loss": 1.5067,
+      "theoretical_loss": 3.472554815091906,
+      "tokens_seen": 1731002368
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048363636363636366,
+      "loss": 1.4398,
+      "theoretical_loss": 3.472543539713277,
+      "tokens_seen": 1731067904
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004836161616161616,
+      "loss": 1.2717,
+      "theoretical_loss": 3.4725322648810297,
+      "tokens_seen": 1731133440
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048359595959595957,
+      "loss": 1.3783,
+      "theoretical_loss": 3.4725209905951173,
+      "tokens_seen": 1731198976
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048357575757575764,
+      "loss": 1.5397,
+      "theoretical_loss": 3.472509716855493,
+      "tokens_seen": 1731264512
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004835555555555556,
+      "loss": 1.4858,
+      "theoretical_loss": 3.4724984436621087,
+      "tokens_seen": 1731330048
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048353535353535355,
+      "loss": 1.3623,
+      "theoretical_loss": 3.4724871710149183,
+      "tokens_seen": 1731395584
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004835151515151515,
+      "loss": 1.3471,
+      "theoretical_loss": 3.4724758989138738,
+      "tokens_seen": 1731461120
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048349494949494946,
+      "loss": 1.5417,
+      "theoretical_loss": 3.472464627358929,
+      "tokens_seen": 1731526656
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048347474747474753,
+      "loss": 1.6138,
+      "theoretical_loss": 3.4724533563500355,
+      "tokens_seen": 1731592192
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004834545454545455,
+      "loss": 1.6054,
+      "theoretical_loss": 3.472442085887147,
+      "tokens_seen": 1731657728
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048343434343434344,
+      "loss": 1.6946,
+      "theoretical_loss": 3.472430815970217,
+      "tokens_seen": 1731723264
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004834141414141414,
+      "loss": 1.4545,
+      "theoretical_loss": 3.4724195465991974,
+      "tokens_seen": 1731788800
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004833939393939394,
+      "loss": 1.4118,
+      "theoretical_loss": 3.4724082777740413,
+      "tokens_seen": 1731854336
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004833737373737374,
+      "loss": 1.2402,
+      "theoretical_loss": 3.472397009494702,
+      "tokens_seen": 1731919872
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004833535353535354,
+      "loss": 1.5217,
+      "theoretical_loss": 3.4723857417611317,
+      "tokens_seen": 1731985408
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/aligned_token_count": 0.37738269567489624,
+      "objective/train/avg_token_score": 0.007554531563073397,
+      "objective/train/docs_used": 980288,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.1984022855758667,
+      "objective/train/original_loss": 4.563331127166748,
+      "objective/train/theoretical_loss": 3.4723744745732836,
+      "objective/train/tokens_used": 90910176,
+      "objective/train/unlikelihood_loss": 0.06496313959360123,
+      "theoretical_loss": 3.4723744745732836,
+      "tokens_seen": 1732050944
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048333333333333334,
+      "loss": 1.1324,
+      "theoretical_loss": 3.4723744745732836,
+      "tokens_seen": 1732050944
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004833131313131313,
+      "loss": 1.2802,
+      "theoretical_loss": 3.472363207931111,
+      "tokens_seen": 1732116480
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004832929292929293,
+      "loss": 1.3541,
+      "theoretical_loss": 3.4723519418345665,
+      "tokens_seen": 1732182016
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004832727272727273,
+      "loss": 1.5818,
+      "theoretical_loss": 3.4723406762836033,
+      "tokens_seen": 1732247552
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048325252525252527,
+      "loss": 1.4111,
+      "theoretical_loss": 3.472329411278174,
+      "tokens_seen": 1732313088
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048323232323232323,
+      "loss": 1.4464,
+      "theoretical_loss": 3.472318146818232,
+      "tokens_seen": 1732378624
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048321212121212124,
+      "loss": 1.4059,
+      "theoretical_loss": 3.4723068829037294,
+      "tokens_seen": 1732444160
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004831919191919192,
+      "loss": 1.2797,
+      "theoretical_loss": 3.47229561953462,
+      "tokens_seen": 1732509696
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004831717171717172,
+      "loss": 1.3734,
+      "theoretical_loss": 3.4722843567108566,
+      "tokens_seen": 1732575232
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048315151515151516,
+      "loss": 1.3392,
+      "theoretical_loss": 3.4722730944323916,
+      "tokens_seen": 1732640768
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004831313131313131,
+      "loss": 1.4454,
+      "theoretical_loss": 3.4722618326991785,
+      "tokens_seen": 1732706304
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048311111111111113,
+      "loss": 1.578,
+      "theoretical_loss": 3.4722505715111707,
+      "tokens_seen": 1732771840
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004830909090909091,
+      "loss": 1.3528,
+      "theoretical_loss": 3.4722393108683205,
+      "tokens_seen": 1732837376
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004830707070707071,
+      "loss": 1.419,
+      "theoretical_loss": 3.472228050770581,
+      "tokens_seen": 1732902912
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048305050505050506,
+      "loss": 1.3326,
+      "theoretical_loss": 3.472216791217905,
+      "tokens_seen": 1732968448
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048303030303030307,
+      "loss": 1.6869,
+      "theoretical_loss": 3.4722055322102463,
+      "tokens_seen": 1733033984
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000483010101010101,
+      "loss": 1.5042,
+      "theoretical_loss": 3.472194273747557,
+      "tokens_seen": 1733099520
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000482989898989899,
+      "loss": 1.3897,
+      "theoretical_loss": 3.47218301582979,
+      "tokens_seen": 1733165056
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.000482969696969697,
+      "loss": 1.2143,
+      "theoretical_loss": 3.4721717584569003,
+      "tokens_seen": 1733230592
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048294949494949495,
+      "loss": 1.5147,
+      "theoretical_loss": 3.472160501628838,
+      "tokens_seen": 1733296128
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048292929292929296,
+      "loss": 1.3647,
+      "theoretical_loss": 3.4721492453455585,
+      "tokens_seen": 1733361664
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004829090909090909,
+      "loss": 1.1661,
+      "theoretical_loss": 3.4721379896070133,
+      "tokens_seen": 1733427200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004828888888888889,
+      "loss": 1.5285,
+      "theoretical_loss": 3.4721267344131563,
+      "tokens_seen": 1733492736
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004828686868686869,
+      "loss": 1.4618,
+      "theoretical_loss": 3.4721154797639406,
+      "tokens_seen": 1733558272
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004828484848484849,
+      "loss": 1.5015,
+      "theoretical_loss": 3.4721042256593186,
+      "tokens_seen": 1733623808
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/aligned_token_count": 0.5816226601600647,
+      "objective/train/avg_token_score": 0.014024200849235058,
+      "objective/train/docs_used": 981887,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 2.1395461559295654,
+      "objective/train/original_loss": 4.308298587799072,
+      "objective/train/theoretical_loss": 3.472092972099244,
+      "objective/train/tokens_used": 92548576,
+      "objective/train/unlikelihood_loss": 0.05324159935116768,
+      "theoretical_loss": 3.472092972099244,
+      "tokens_seen": 1733689344
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048282828282828285,
+      "loss": 1.4546,
+      "theoretical_loss": 3.472092972099244,
+      "tokens_seen": 1733689344
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004828080808080808,
+      "loss": 1.1628,
+      "theoretical_loss": 3.4720817190836692,
+      "tokens_seen": 1733754880
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048278787878787877,
+      "loss": 1.2467,
+      "theoretical_loss": 3.472070466612548,
+      "tokens_seen": 1733820416
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004827676767676768,
+      "loss": 1.4641,
+      "theoretical_loss": 3.4720592146858333,
+      "tokens_seen": 1733885952
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004827474747474748,
+      "loss": 1.5125,
+      "theoretical_loss": 3.4720479633034778,
+      "tokens_seen": 1733951488
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048272727272727275,
+      "loss": 1.3324,
+      "theoretical_loss": 3.472036712465435,
+      "tokens_seen": 1734017024
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004827070707070707,
+      "loss": 1.3651,
+      "theoretical_loss": 3.4720254621716578,
+      "tokens_seen": 1734082560
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048268686868686866,
+      "loss": 1.4273,
+      "theoretical_loss": 3.472014212422099,
+      "tokens_seen": 1734148096
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048266666666666667,
+      "loss": 1.2076,
+      "theoretical_loss": 3.4720029632167124,
+      "tokens_seen": 1734213632
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004826464646464647,
+      "loss": 1.1455,
+      "theoretical_loss": 3.471991714555451,
+      "tokens_seen": 1734279168
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048262626262626264,
+      "loss": 1.5309,
+      "theoretical_loss": 3.471980466438267,
+      "tokens_seen": 1734344704
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004826060606060606,
+      "loss": 1.5117,
+      "theoretical_loss": 3.471969218865115,
+      "tokens_seen": 1734410240
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048258585858585855,
+      "loss": 1.5216,
+      "theoretical_loss": 3.4719579718359466,
+      "tokens_seen": 1734475776
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004825656565656566,
+      "loss": 1.52,
+      "theoretical_loss": 3.4719467253507164,
+      "tokens_seen": 1734541312
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004825454545454546,
+      "loss": 1.5335,
+      "theoretical_loss": 3.471935479409376,
+      "tokens_seen": 1734606848
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048252525252525253,
+      "loss": 1.531,
+      "theoretical_loss": 3.4719242340118797,
+      "tokens_seen": 1734672384
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004825050505050505,
+      "loss": 1.3912,
+      "theoretical_loss": 3.471912989158181,
+      "tokens_seen": 1734737920
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048248484848484845,
+      "loss": 1.5328,
+      "theoretical_loss": 3.471901744848231,
+      "tokens_seen": 1734803456
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004824646464646465,
+      "loss": 1.4363,
+      "theoretical_loss": 3.471890501081985,
+      "tokens_seen": 1734868992
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048244444444444447,
+      "loss": 1.6167,
+      "theoretical_loss": 3.4718792578593956,
+      "tokens_seen": 1734934528
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004824242424242424,
+      "loss": 1.3632,
+      "theoretical_loss": 3.4718680151804158,
+      "tokens_seen": 1735000064
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004824040404040404,
+      "loss": 1.3851,
+      "theoretical_loss": 3.4718567730449985,
+      "tokens_seen": 1735065600
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004823838383838384,
+      "loss": 1.2801,
+      "theoretical_loss": 3.4718455314530976,
+      "tokens_seen": 1735131136
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004823636363636364,
+      "loss": 1.3273,
+      "theoretical_loss": 3.471834290404665,
+      "tokens_seen": 1735196672
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00048234343434343436,
+      "loss": 1.5982,
+      "theoretical_loss": 3.4718230498996556,
+      "tokens_seen": 1735262208
+    },
+    {
+      "epoch": 0.04,
+      "objective/train/aligned_token_count": 0.25384896993637085,
+      "objective/train/avg_token_score": 0.01493894774466753,
+      "objective/train/docs_used": 982466,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 0.8242332339286804,
+      "objective/train/original_loss": 4.889101505279541,
+      "objective/train/theoretical_loss": 3.4718118099380213,
+      "objective/train/tokens_used": 94186976,
+      "objective/train/unlikelihood_loss": 0.03157869726419449,
+      "theoretical_loss": 3.4718118099380213,
+      "tokens_seen": 1735327744
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.0004823232323232323,
+      "loss": 1.1415,
+      "theoretical_loss": 3.4718118099380213,
+      "tokens_seen": 1735327744
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004823030303030303,
+      "loss": 1.321,
+      "theoretical_loss": 3.4718005705197164,
+      "tokens_seen": 1735393280
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004822828282828283,
+      "loss": 1.6192,
+      "theoretical_loss": 3.4717893316446933,
+      "tokens_seen": 1735458816
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004822626262626263,
+      "loss": 1.277,
+      "theoretical_loss": 3.471778093312905,
+      "tokens_seen": 1735524352
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048224242424242425,
+      "loss": 1.3975,
+      "theoretical_loss": 3.4717668555243053,
+      "tokens_seen": 1735589888
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004822222222222222,
+      "loss": 1.5485,
+      "theoretical_loss": 3.4717556182788476,
+      "tokens_seen": 1735655424
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004822020202020202,
+      "loss": 1.1766,
+      "theoretical_loss": 3.471744381576485,
+      "tokens_seen": 1735720960
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004821818181818182,
+      "loss": 1.3168,
+      "theoretical_loss": 3.4717331454171703,
+      "tokens_seen": 1735786496
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004821616161616162,
+      "loss": 1.4985,
+      "theoretical_loss": 3.471721909800857,
+      "tokens_seen": 1735852032
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048214141414141415,
+      "loss": 1.4077,
+      "theoretical_loss": 3.4717106747274986,
+      "tokens_seen": 1735917568
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004821212121212121,
+      "loss": 1.4942,
+      "theoretical_loss": 3.471699440197048,
+      "tokens_seen": 1735983104
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004821010101010101,
+      "loss": 1.5322,
+      "theoretical_loss": 3.4716882062094587,
+      "tokens_seen": 1736048640
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048208080808080807,
+      "loss": 1.4188,
+      "theoretical_loss": 3.4716769727646835,
+      "tokens_seen": 1736114176
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004820606060606061,
+      "loss": 1.5154,
+      "theoretical_loss": 3.4716657398626767,
+      "tokens_seen": 1736179712
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048204040404040404,
+      "loss": 1.2208,
+      "theoretical_loss": 3.471654507503391,
+      "tokens_seen": 1736245248
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048202020202020205,
+      "loss": 1.524,
+      "theoretical_loss": 3.4716432756867794,
+      "tokens_seen": 1736310784
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000482,
+      "loss": 1.5759,
+      "theoretical_loss": 3.4716320444127957,
+      "tokens_seen": 1736376320
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048197979797979796,
+      "loss": 1.3482,
+      "theoretical_loss": 3.471620813681393,
+      "tokens_seen": 1736441856
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000481959595959596,
+      "loss": 1.4318,
+      "theoretical_loss": 3.4716095834925245,
+      "tokens_seen": 1736507392
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048193939393939393,
+      "loss": 1.423,
+      "theoretical_loss": 3.4715983538461432,
+      "tokens_seen": 1736572928
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048191919191919194,
+      "loss": 1.4259,
+      "theoretical_loss": 3.4715871247422037,
+      "tokens_seen": 1736638464
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004818989898989899,
+      "loss": 1.6113,
+      "theoretical_loss": 3.4715758961806573,
+      "tokens_seen": 1736704000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004818787878787879,
+      "loss": 1.4458,
+      "theoretical_loss": 3.4715646681614594,
+      "tokens_seen": 1736769536
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048185858585858587,
+      "loss": 1.1604,
+      "theoretical_loss": 3.4715534406845627,
+      "tokens_seen": 1736835072
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004818383838383839,
+      "loss": 1.3382,
+      "theoretical_loss": 3.4715422137499194,
+      "tokens_seen": 1736900608
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/aligned_token_count": 0.43413978815078735,
+      "objective/train/avg_token_score": 0.012802019715309143,
+      "objective/train/docs_used": 983872,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.4975614547729492,
+      "objective/train/original_loss": 4.492546081542969,
+      "objective/train/theoretical_loss": 3.4715309873574847,
+      "objective/train/tokens_used": 95825376,
+      "objective/train/unlikelihood_loss": 0.04530109092593193,
+      "theoretical_loss": 3.4715309873574847,
+      "tokens_seen": 1736966144
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048181818181818184,
+      "loss": 1.3615,
+      "theoretical_loss": 3.4715309873574847,
+      "tokens_seen": 1736966144
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004817979797979798,
+      "loss": 1.5586,
+      "theoretical_loss": 3.4715197615072104,
+      "tokens_seen": 1737031680
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004817777777777778,
+      "loss": 1.5499,
+      "theoretical_loss": 3.4715085361990505,
+      "tokens_seen": 1737097216
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048175757575757576,
+      "loss": 1.3085,
+      "theoretical_loss": 3.471497311432959,
+      "tokens_seen": 1737162752
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048173737373737377,
+      "loss": 1.4672,
+      "theoretical_loss": 3.4714860872088877,
+      "tokens_seen": 1737228288
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048171717171717173,
+      "loss": 1.3109,
+      "theoretical_loss": 3.471474863526791,
+      "tokens_seen": 1737293824
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004816969696969697,
+      "loss": 1.5911,
+      "theoretical_loss": 3.471463640386623,
+      "tokens_seen": 1737359360
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004816767676767677,
+      "loss": 1.4829,
+      "theoretical_loss": 3.4714524177883352,
+      "tokens_seen": 1737424896
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004816565656565657,
+      "loss": 1.3043,
+      "theoretical_loss": 3.4714411957318827,
+      "tokens_seen": 1737490432
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048163636363636366,
+      "loss": 1.5117,
+      "theoretical_loss": 3.4714299742172186,
+      "tokens_seen": 1737555968
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004816161616161616,
+      "loss": 1.5361,
+      "theoretical_loss": 3.4714187532442953,
+      "tokens_seen": 1737621504
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004815959595959596,
+      "loss": 1.4369,
+      "theoretical_loss": 3.471407532813067,
+      "tokens_seen": 1737687040
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004815757575757576,
+      "loss": 1.3458,
+      "theoretical_loss": 3.4713963129234875,
+      "tokens_seen": 1737752576
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004815555555555556,
+      "loss": 1.4904,
+      "theoretical_loss": 3.4713850935755093,
+      "tokens_seen": 1737818112
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048153535353535356,
+      "loss": 1.3929,
+      "theoretical_loss": 3.4713738747690863,
+      "tokens_seen": 1737883648
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004815151515151515,
+      "loss": 1.5227,
+      "theoretical_loss": 3.471362656504172,
+      "tokens_seen": 1737949184
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048149494949494947,
+      "loss": 1.4509,
+      "theoretical_loss": 3.4713514387807196,
+      "tokens_seen": 1738014720
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048147474747474754,
+      "loss": 1.3549,
+      "theoretical_loss": 3.471340221598683,
+      "tokens_seen": 1738080256
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004814545454545455,
+      "loss": 1.4395,
+      "theoretical_loss": 3.4713290049580157,
+      "tokens_seen": 1738145792
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048143434343434345,
+      "loss": 1.484,
+      "theoretical_loss": 3.4713177888586704,
+      "tokens_seen": 1738211328
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004814141414141414,
+      "loss": 1.3216,
+      "theoretical_loss": 3.471306573300601,
+      "tokens_seen": 1738276864
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048139393939393936,
+      "loss": 1.6081,
+      "theoretical_loss": 3.4712953582837613,
+      "tokens_seen": 1738342400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048137373737373743,
+      "loss": 1.5769,
+      "theoretical_loss": 3.471284143808104,
+      "tokens_seen": 1738407936
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004813535353535354,
+      "loss": 1.4718,
+      "theoretical_loss": 3.4712729298735834,
+      "tokens_seen": 1738473472
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048133333333333334,
+      "loss": 1.3028,
+      "theoretical_loss": 3.471261716480152,
+      "tokens_seen": 1738539008
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/aligned_token_count": 0.48277124762535095,
+      "objective/train/avg_token_score": 0.009705035015940666,
+      "objective/train/docs_used": 984348,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.655248999595642,
+      "objective/train/original_loss": 4.150419235229492,
+      "objective/train/theoretical_loss": 3.4712505036277648,
+      "objective/train/tokens_used": 97463776,
+      "objective/train/unlikelihood_loss": 0.052979037165641785,
+      "theoretical_loss": 3.4712505036277648,
+      "tokens_seen": 1738604544
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004813131313131313,
+      "loss": 1.4086,
+      "theoretical_loss": 3.4712505036277648,
+      "tokens_seen": 1738604544
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048129292929292926,
+      "loss": 1.3672,
+      "theoretical_loss": 3.4712392913163734,
+      "tokens_seen": 1738670080
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004812727272727273,
+      "loss": 1.4242,
+      "theoretical_loss": 3.4712280795459334,
+      "tokens_seen": 1738735616
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004812525252525253,
+      "loss": 1.5003,
+      "theoretical_loss": 3.4712168683163966,
+      "tokens_seen": 1738801152
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048123232323232323,
+      "loss": 1.3127,
+      "theoretical_loss": 3.4712056576277175,
+      "tokens_seen": 1738866688
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004812121212121212,
+      "loss": 1.3313,
+      "theoretical_loss": 3.4711944474798493,
+      "tokens_seen": 1738932224
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004811919191919192,
+      "loss": 1.3412,
+      "theoretical_loss": 3.4711832378727454,
+      "tokens_seen": 1738997760
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004811717171717172,
+      "loss": 1.3733,
+      "theoretical_loss": 3.4711720288063592,
+      "tokens_seen": 1739063296
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048115151515151517,
+      "loss": 1.4347,
+      "theoretical_loss": 3.4711608202806445,
+      "tokens_seen": 1739128832
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048113131313131313,
+      "loss": 1.1575,
+      "theoretical_loss": 3.471149612295555,
+      "tokens_seen": 1739194368
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004811111111111111,
+      "loss": 1.4967,
+      "theoretical_loss": 3.4711384048510445,
+      "tokens_seen": 1739259904
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004810909090909091,
+      "loss": 1.5965,
+      "theoretical_loss": 3.471127197947066,
+      "tokens_seen": 1739325440
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004810707070707071,
+      "loss": 1.6453,
+      "theoretical_loss": 3.4711159915835728,
+      "tokens_seen": 1739390976
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048105050505050506,
+      "loss": 1.4245,
+      "theoretical_loss": 3.4711047857605193,
+      "tokens_seen": 1739456512
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000481030303030303,
+      "loss": 1.5783,
+      "theoretical_loss": 3.4710935804778584,
+      "tokens_seen": 1739522048
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048101010101010103,
+      "loss": 1.2761,
+      "theoretical_loss": 3.471082375735544,
+      "tokens_seen": 1739587584
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000480989898989899,
+      "loss": 1.4248,
+      "theoretical_loss": 3.47107117153353,
+      "tokens_seen": 1739653120
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000480969696969697,
+      "loss": 1.6432,
+      "theoretical_loss": 3.471059967871769,
+      "tokens_seen": 1739718656
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048094949494949496,
+      "loss": 1.394,
+      "theoretical_loss": 3.4710487647502157,
+      "tokens_seen": 1739784192
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004809292929292929,
+      "loss": 1.3901,
+      "theoretical_loss": 3.4710375621688234,
+      "tokens_seen": 1739849728
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004809090909090909,
+      "loss": 1.4413,
+      "theoretical_loss": 3.471026360127545,
+      "tokens_seen": 1739915264
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004808888888888889,
+      "loss": 1.4342,
+      "theoretical_loss": 3.471015158626335,
+      "tokens_seen": 1739980800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004808686868686869,
+      "loss": 1.2352,
+      "theoretical_loss": 3.471003957665147,
+      "tokens_seen": 1740046336
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048084848484848485,
+      "loss": 1.2348,
+      "theoretical_loss": 3.470992757243934,
+      "tokens_seen": 1740111872
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048082828282828286,
+      "loss": 1.3523,
+      "theoretical_loss": 3.4709815573626495,
+      "tokens_seen": 1740177408
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/aligned_token_count": 0.33125612139701843,
+      "objective/train/avg_token_score": 0.017105352133512497,
+      "objective/train/docs_used": 985100,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.2295209169387817,
+      "objective/train/original_loss": 4.453027248382568,
+      "objective/train/theoretical_loss": 3.4709703580212485,
+      "objective/train/tokens_used": 99102176,
+      "objective/train/unlikelihood_loss": 0.0723608136177063,
+      "theoretical_loss": 3.4709703580212485,
+      "tokens_seen": 1740242944
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004808080808080808,
+      "loss": 1.4608,
+      "theoretical_loss": 3.4709703580212485,
+      "tokens_seen": 1740242944
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004807878787878788,
+      "loss": 1.4108,
+      "theoretical_loss": 3.470959159219683,
+      "tokens_seen": 1740308480
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004807676767676768,
+      "loss": 1.5332,
+      "theoretical_loss": 3.4709479609579077,
+      "tokens_seen": 1740374016
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048074747474747474,
+      "loss": 1.7023,
+      "theoretical_loss": 3.470936763235876,
+      "tokens_seen": 1740439552
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048072727272727275,
+      "loss": 1.24,
+      "theoretical_loss": 3.470925566053541,
+      "tokens_seen": 1740505088
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004807070707070707,
+      "loss": 1.4697,
+      "theoretical_loss": 3.4709143694108575,
+      "tokens_seen": 1740570624
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048068686868686867,
+      "loss": 1.4578,
+      "theoretical_loss": 3.4709031733077786,
+      "tokens_seen": 1740636160
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004806666666666667,
+      "loss": 1.5702,
+      "theoretical_loss": 3.4708919777442575,
+      "tokens_seen": 1740701696
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004806464646464647,
+      "loss": 1.5591,
+      "theoretical_loss": 3.470880782720249,
+      "tokens_seen": 1740767232
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048062626262626264,
+      "loss": 1.2108,
+      "theoretical_loss": 3.470869588235705,
+      "tokens_seen": 1740832768
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004806060606060606,
+      "loss": 1.3671,
+      "theoretical_loss": 3.4708583942905813,
+      "tokens_seen": 1740898304
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048058585858585856,
+      "loss": 1.5984,
+      "theoretical_loss": 3.4708472008848297,
+      "tokens_seen": 1740963840
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048056565656565657,
+      "loss": 1.5255,
+      "theoretical_loss": 3.4708360080184057,
+      "tokens_seen": 1741029376
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004805454545454546,
+      "loss": 1.5005,
+      "theoretical_loss": 3.4708248156912616,
+      "tokens_seen": 1741094912
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048052525252525254,
+      "loss": 1.5318,
+      "theoretical_loss": 3.470813623903352,
+      "tokens_seen": 1741160448
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004805050505050505,
+      "loss": 1.3555,
+      "theoretical_loss": 3.47080243265463,
+      "tokens_seen": 1741225984
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048048484848484845,
+      "loss": 1.324,
+      "theoretical_loss": 3.47079124194505,
+      "tokens_seen": 1741291520
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004804646464646465,
+      "loss": 1.4643,
+      "theoretical_loss": 3.4707800517745646,
+      "tokens_seen": 1741357056
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004804444444444445,
+      "loss": 1.2959,
+      "theoretical_loss": 3.470768862143129,
+      "tokens_seen": 1741422592
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048042424242424243,
+      "loss": 1.3676,
+      "theoretical_loss": 3.470757673050696,
+      "tokens_seen": 1741488128
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004804040404040404,
+      "loss": 1.4532,
+      "theoretical_loss": 3.4707464844972193,
+      "tokens_seen": 1741553664
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048038383838383834,
+      "loss": 1.3073,
+      "theoretical_loss": 3.470735296482653,
+      "tokens_seen": 1741619200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004803636363636364,
+      "loss": 1.3783,
+      "theoretical_loss": 3.470724109006951,
+      "tokens_seen": 1741684736
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048034343434343437,
+      "loss": 1.6342,
+      "theoretical_loss": 3.470712922070067,
+      "tokens_seen": 1741750272
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004803232323232323,
+      "loss": 1.5602,
+      "theoretical_loss": 3.4707017356719545,
+      "tokens_seen": 1741815808
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/aligned_token_count": 0.4005987346172333,
+      "objective/train/avg_token_score": 0.019556434825062752,
+      "objective/train/docs_used": 986266,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.536611795425415,
+      "objective/train/original_loss": 4.283116817474365,
+      "objective/train/theoretical_loss": 3.4706905498125673,
+      "objective/train/tokens_used": 100740576,
+      "objective/train/unlikelihood_loss": 0.08906306326389313,
+      "theoretical_loss": 3.4706905498125673,
+      "tokens_seen": 1741881344
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004803030303030303,
+      "loss": 1.4406,
+      "theoretical_loss": 3.4706905498125673,
+      "tokens_seen": 1741881344
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004802828282828283,
+      "loss": 1.4997,
+      "theoretical_loss": 3.47067936449186,
+      "tokens_seen": 1741946880
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004802626262626263,
+      "loss": 1.6152,
+      "theoretical_loss": 3.4706681797097847,
+      "tokens_seen": 1742012416
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048024242424242426,
+      "loss": 1.5295,
+      "theoretical_loss": 3.470656995466297,
+      "tokens_seen": 1742077952
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004802222222222222,
+      "loss": 1.3141,
+      "theoretical_loss": 3.47064581176135,
+      "tokens_seen": 1742143488
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048020202020202017,
+      "loss": 1.3606,
+      "theoretical_loss": 3.470634628594897,
+      "tokens_seen": 1742209024
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004801818181818182,
+      "loss": 1.4034,
+      "theoretical_loss": 3.4706234459668925,
+      "tokens_seen": 1742274560
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004801616161616162,
+      "loss": 1.4074,
+      "theoretical_loss": 3.47061226387729,
+      "tokens_seen": 1742340096
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048014141414141415,
+      "loss": 1.2829,
+      "theoretical_loss": 3.470601082326043,
+      "tokens_seen": 1742405632
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004801212121212121,
+      "loss": 1.4758,
+      "theoretical_loss": 3.4705899013131063,
+      "tokens_seen": 1742471168
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004801010101010101,
+      "loss": 1.3614,
+      "theoretical_loss": 3.470578720838433,
+      "tokens_seen": 1742536704
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048008080808080813,
+      "loss": 1.3864,
+      "theoretical_loss": 3.4705675409019774,
+      "tokens_seen": 1742602240
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004800606060606061,
+      "loss": 1.4217,
+      "theoretical_loss": 3.4705563615036925,
+      "tokens_seen": 1742667776
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048004040404040404,
+      "loss": 1.1732,
+      "theoretical_loss": 3.4705451826435327,
+      "tokens_seen": 1742733312
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000480020202020202,
+      "loss": 1.4523,
+      "theoretical_loss": 3.4705340043214523,
+      "tokens_seen": 1742798848
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00048,
+      "loss": 1.4718,
+      "theoretical_loss": 3.4705228265374046,
+      "tokens_seen": 1742864384
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000479979797979798,
+      "loss": 1.4891,
+      "theoretical_loss": 3.470511649291344,
+      "tokens_seen": 1742929920
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000479959595959596,
+      "loss": 1.2984,
+      "theoretical_loss": 3.4705004725832236,
+      "tokens_seen": 1742995456
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047993939393939394,
+      "loss": 1.5396,
+      "theoretical_loss": 3.470489296412998,
+      "tokens_seen": 1743060992
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047991919191919195,
+      "loss": 1.6143,
+      "theoretical_loss": 3.470478120780621,
+      "tokens_seen": 1743126528
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004798989898989899,
+      "loss": 1.3893,
+      "theoretical_loss": 3.4704669456860455,
+      "tokens_seen": 1743192064
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004798787878787879,
+      "loss": 1.3921,
+      "theoretical_loss": 3.4704557711292265,
+      "tokens_seen": 1743257600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047985858585858587,
+      "loss": 1.326,
+      "theoretical_loss": 3.470444597110118,
+      "tokens_seen": 1743323136
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047983838383838383,
+      "loss": 1.3948,
+      "theoretical_loss": 3.4704334236286734,
+      "tokens_seen": 1743388672
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047981818181818184,
+      "loss": 1.3807,
+      "theoretical_loss": 3.4704222506848463,
+      "tokens_seen": 1743454208
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/aligned_token_count": 0.3908846378326416,
+      "objective/train/avg_token_score": 0.00899689830839634,
+      "objective/train/docs_used": 986906,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.2534172534942627,
+      "objective/train/original_loss": 4.556232929229736,
+      "objective/train/theoretical_loss": 3.470411078278591,
+      "objective/train/tokens_used": 102378976,
+      "objective/train/unlikelihood_loss": 0.03879563882946968,
+      "theoretical_loss": 3.470411078278591,
+      "tokens_seen": 1743519744
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004797979797979798,
+      "loss": 1.5736,
+      "theoretical_loss": 3.470411078278591,
+      "tokens_seen": 1743519744
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004797777777777778,
+      "loss": 1.304,
+      "theoretical_loss": 3.4703999064098623,
+      "tokens_seen": 1743585280
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047975757575757577,
+      "loss": 1.4849,
+      "theoretical_loss": 3.470388735078613,
+      "tokens_seen": 1743650816
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004797373737373738,
+      "loss": 1.3432,
+      "theoretical_loss": 3.4703775642847967,
+      "tokens_seen": 1743716352
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047971717171717173,
+      "loss": 1.1625,
+      "theoretical_loss": 3.4703663940283684,
+      "tokens_seen": 1743781888
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004796969696969697,
+      "loss": 1.6368,
+      "theoretical_loss": 3.470355224309282,
+      "tokens_seen": 1743847424
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004796767676767677,
+      "loss": 1.4579,
+      "theoretical_loss": 3.470344055127491,
+      "tokens_seen": 1743912960
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047965656565656566,
+      "loss": 1.4415,
+      "theoretical_loss": 3.4703328864829492,
+      "tokens_seen": 1743978496
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047963636363636367,
+      "loss": 1.5911,
+      "theoretical_loss": 3.4703217183756108,
+      "tokens_seen": 1744044032
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004796161616161616,
+      "loss": 1.2517,
+      "theoretical_loss": 3.47031055080543,
+      "tokens_seen": 1744109568
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004795959595959596,
+      "loss": 1.2024,
+      "theoretical_loss": 3.470299383772361,
+      "tokens_seen": 1744175104
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004795757575757576,
+      "loss": 1.3713,
+      "theoretical_loss": 3.470288217276357,
+      "tokens_seen": 1744240640
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004795555555555556,
+      "loss": 1.4253,
+      "theoretical_loss": 3.4702770513173724,
+      "tokens_seen": 1744306176
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047953535353535356,
+      "loss": 1.406,
+      "theoretical_loss": 3.470265885895361,
+      "tokens_seen": 1744371712
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004795151515151515,
+      "loss": 1.4899,
+      "theoretical_loss": 3.4702547210102774,
+      "tokens_seen": 1744437248
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004794949494949495,
+      "loss": 1.5386,
+      "theoretical_loss": 3.470243556662075,
+      "tokens_seen": 1744502784
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004794747474747475,
+      "loss": 1.2509,
+      "theoretical_loss": 3.470232392850708,
+      "tokens_seen": 1744568320
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004794545454545455,
+      "loss": 1.7622,
+      "theoretical_loss": 3.470221229576131,
+      "tokens_seen": 1744633856
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047943434343434345,
+      "loss": 1.3678,
+      "theoretical_loss": 3.4702100668382965,
+      "tokens_seen": 1744699392
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004794141414141414,
+      "loss": 1.4218,
+      "theoretical_loss": 3.47019890463716,
+      "tokens_seen": 1744764928
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047939393939393937,
+      "loss": 1.4213,
+      "theoretical_loss": 3.470187742972675,
+      "tokens_seen": 1744830464
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004793737373737374,
+      "loss": 1.4785,
+      "theoretical_loss": 3.470176581844796,
+      "tokens_seen": 1744896000
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004793535353535354,
+      "loss": 1.5449,
+      "theoretical_loss": 3.4701654212534763,
+      "tokens_seen": 1744961536
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047933333333333335,
+      "loss": 1.287,
+      "theoretical_loss": 3.4701542611986698,
+      "tokens_seen": 1745027072
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004793131313131313,
+      "loss": 1.567,
+      "theoretical_loss": 3.470143101680332,
+      "tokens_seen": 1745092608
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/aligned_token_count": 0.2597140669822693,
+      "objective/train/avg_token_score": 0.04489205777645111,
+      "objective/train/docs_used": 988319,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 0.912868082523346,
+      "objective/train/original_loss": 4.443542003631592,
+      "objective/train/theoretical_loss": 3.4701319426984156,
+      "objective/train/tokens_used": 104017376,
+      "objective/train/unlikelihood_loss": 0.06317847967147827,
+      "theoretical_loss": 3.4701319426984156,
+      "tokens_seen": 1745158144
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047929292929292926,
+      "loss": 1.2568,
+      "theoretical_loss": 3.4701319426984156,
+      "tokens_seen": 1745158144
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004792727272727273,
+      "loss": 1.3641,
+      "theoretical_loss": 3.4701207842528747,
+      "tokens_seen": 1745223680
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004792525252525253,
+      "loss": 1.5416,
+      "theoretical_loss": 3.470109626343664,
+      "tokens_seen": 1745289216
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047923232323232324,
+      "loss": 1.4213,
+      "theoretical_loss": 3.470098468970738,
+      "tokens_seen": 1745354752
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004792121212121212,
+      "loss": 1.0977,
+      "theoretical_loss": 3.470087312134049,
+      "tokens_seen": 1745420288
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047919191919191915,
+      "loss": 1.5069,
+      "theoretical_loss": 3.470076155833553,
+      "tokens_seen": 1745485824
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004791717171717172,
+      "loss": 1.569,
+      "theoretical_loss": 3.4700650000692033,
+      "tokens_seen": 1745551360
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004791515151515152,
+      "loss": 1.5586,
+      "theoretical_loss": 3.4700538448409537,
+      "tokens_seen": 1745616896
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047913131313131313,
+      "loss": 1.5639,
+      "theoretical_loss": 3.470042690148759,
+      "tokens_seen": 1745682432
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004791111111111111,
+      "loss": 1.4329,
+      "theoretical_loss": 3.4700315359925726,
+      "tokens_seen": 1745747968
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004790909090909091,
+      "loss": 1.4756,
+      "theoretical_loss": 3.470020382372349,
+      "tokens_seen": 1745813504
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004790707070707071,
+      "loss": 1.4192,
+      "theoretical_loss": 3.470009229288043,
+      "tokens_seen": 1745879040
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047905050505050507,
+      "loss": 1.2349,
+      "theoretical_loss": 3.469998076739607,
+      "tokens_seen": 1745944576
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000479030303030303,
+      "loss": 1.4639,
+      "theoretical_loss": 3.469986924726997,
+      "tokens_seen": 1746010112
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000479010101010101,
+      "loss": 1.5232,
+      "theoretical_loss": 3.469975773250166,
+      "tokens_seen": 1746075648
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000478989898989899,
+      "loss": 1.1636,
+      "theoretical_loss": 3.4699646223090688,
+      "tokens_seen": 1746141184
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000478969696969697,
+      "loss": 1.4086,
+      "theoretical_loss": 3.469953471903659,
+      "tokens_seen": 1746206720
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047894949494949496,
+      "loss": 1.3567,
+      "theoretical_loss": 3.469942322033891,
+      "tokens_seen": 1746272256
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004789292929292929,
+      "loss": 1.6627,
+      "theoretical_loss": 3.4699311726997193,
+      "tokens_seen": 1746337792
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047890909090909093,
+      "loss": 1.4643,
+      "theoretical_loss": 3.469920023901097,
+      "tokens_seen": 1746403328
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004788888888888889,
+      "loss": 1.393,
+      "theoretical_loss": 3.46990887563798,
+      "tokens_seen": 1746468864
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004788686868686869,
+      "loss": 1.6576,
+      "theoretical_loss": 3.469897727910321,
+      "tokens_seen": 1746534400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047884848484848485,
+      "loss": 1.3613,
+      "theoretical_loss": 3.4698865807180748,
+      "tokens_seen": 1746599936
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004788282828282828,
+      "loss": 1.5132,
+      "theoretical_loss": 3.4698754340611955,
+      "tokens_seen": 1746665472
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004788080808080808,
+      "loss": 1.4484,
+      "theoretical_loss": 3.4698642879396373,
+      "tokens_seen": 1746731008
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/aligned_token_count": 0.29673752188682556,
+      "objective/train/avg_token_score": 0.007496143225580454,
+      "objective/train/docs_used": 989102,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 0.9727373123168945,
+      "objective/train/original_loss": 4.123487949371338,
+      "objective/train/theoretical_loss": 3.4698531423533545,
+      "objective/train/tokens_used": 105655776,
+      "objective/train/unlikelihood_loss": 0.07145555317401886,
+      "theoretical_loss": 3.4698531423533545,
+      "tokens_seen": 1746796544
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004787878787878788,
+      "loss": 1.2235,
+      "theoretical_loss": 3.4698531423533545,
+      "tokens_seen": 1746796544
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004787676767676768,
+      "loss": 1.4326,
+      "theoretical_loss": 3.469841997302301,
+      "tokens_seen": 1746862080
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047874747474747475,
+      "loss": 1.4788,
+      "theoretical_loss": 3.4698308527864317,
+      "tokens_seen": 1746927616
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047872727272727276,
+      "loss": 1.4569,
+      "theoretical_loss": 3.4698197088056997,
+      "tokens_seen": 1746993152
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004787070707070707,
+      "loss": 1.3571,
+      "theoretical_loss": 3.4698085653600605,
+      "tokens_seen": 1747058688
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047868686868686867,
+      "loss": 1.3078,
+      "theoretical_loss": 3.469797422449467,
+      "tokens_seen": 1747124224
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004786666666666667,
+      "loss": 1.3261,
+      "theoretical_loss": 3.4697862800738752,
+      "tokens_seen": 1747189760
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047864646464646464,
+      "loss": 1.5378,
+      "theoretical_loss": 3.4697751382332376,
+      "tokens_seen": 1747255296
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047862626262626265,
+      "loss": 1.3303,
+      "theoretical_loss": 3.46976399692751,
+      "tokens_seen": 1747320832
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004786060606060606,
+      "loss": 1.513,
+      "theoretical_loss": 3.469752856156645,
+      "tokens_seen": 1747386368
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047858585858585856,
+      "loss": 1.5127,
+      "theoretical_loss": 3.469741715920598,
+      "tokens_seen": 1747451904
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004785656565656566,
+      "loss": 1.511,
+      "theoretical_loss": 3.4697305762193227,
+      "tokens_seen": 1747517440
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004785454545454546,
+      "loss": 1.3687,
+      "theoretical_loss": 3.469719437052774,
+      "tokens_seen": 1747582976
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047852525252525254,
+      "loss": 1.2167,
+      "theoretical_loss": 3.4697082984209056,
+      "tokens_seen": 1747648512
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004785050505050505,
+      "loss": 1.3243,
+      "theoretical_loss": 3.469697160323672,
+      "tokens_seen": 1747714048
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047848484848484846,
+      "loss": 1.7173,
+      "theoretical_loss": 3.469686022761028,
+      "tokens_seen": 1747779584
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047846464646464647,
+      "loss": 1.2967,
+      "theoretical_loss": 3.469674885732927,
+      "tokens_seen": 1747845120
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004784444444444445,
+      "loss": 1.3806,
+      "theoretical_loss": 3.4696637492393236,
+      "tokens_seen": 1747910656
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047842424242424244,
+      "loss": 1.433,
+      "theoretical_loss": 3.4696526132801724,
+      "tokens_seen": 1747976192
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004784040404040404,
+      "loss": 1.2802,
+      "theoretical_loss": 3.4696414778554274,
+      "tokens_seen": 1748041728
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004783838383838384,
+      "loss": 1.3713,
+      "theoretical_loss": 3.4696303429650435,
+      "tokens_seen": 1748107264
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004783636363636364,
+      "loss": 1.288,
+      "theoretical_loss": 3.469619208608974,
+      "tokens_seen": 1748172800
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047834343434343437,
+      "loss": 1.5354,
+      "theoretical_loss": 3.469608074787174,
+      "tokens_seen": 1748238336
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047832323232323233,
+      "loss": 1.3335,
+      "theoretical_loss": 3.4695969414995975,
+      "tokens_seen": 1748303872
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004783030303030303,
+      "loss": 1.2709,
+      "theoretical_loss": 3.4695858087461993,
+      "tokens_seen": 1748369408
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/aligned_token_count": 0.356671541929245,
+      "objective/train/avg_token_score": 0.015191304497420788,
+      "objective/train/docs_used": 990185,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.0629782676696777,
+      "objective/train/original_loss": 4.2517290115356445,
+      "objective/train/theoretical_loss": 3.469574676526933,
+      "objective/train/tokens_used": 107294176,
+      "objective/train/unlikelihood_loss": 0.06087062880396843,
+      "theoretical_loss": 3.469574676526933,
+      "tokens_seen": 1748434944
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004782828282828283,
+      "loss": 1.27,
+      "theoretical_loss": 3.469574676526933,
+      "tokens_seen": 1748434944
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004782626262626263,
+      "loss": 1.3251,
+      "theoretical_loss": 3.469563544841754,
+      "tokens_seen": 1748500480
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047824242424242426,
+      "loss": 1.374,
+      "theoretical_loss": 3.4695524136906153,
+      "tokens_seen": 1748566016
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004782222222222222,
+      "loss": 1.3274,
+      "theoretical_loss": 3.4695412830734726,
+      "tokens_seen": 1748631552
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004782020202020202,
+      "loss": 1.5862,
+      "theoretical_loss": 3.46953015299028,
+      "tokens_seen": 1748697088
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047818181818181824,
+      "loss": 1.3369,
+      "theoretical_loss": 3.469519023440991,
+      "tokens_seen": 1748762624
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004781616161616162,
+      "loss": 1.5887,
+      "theoretical_loss": 3.4695078944255604,
+      "tokens_seen": 1748828160
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047814141414141416,
+      "loss": 1.4068,
+      "theoretical_loss": 3.4694967659439433,
+      "tokens_seen": 1748893696
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004781212121212121,
+      "loss": 1.4508,
+      "theoretical_loss": 3.469485637996093,
+      "tokens_seen": 1748959232
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047810101010101007,
+      "loss": 1.3742,
+      "theoretical_loss": 3.4694745105819647,
+      "tokens_seen": 1749024768
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047808080808080814,
+      "loss": 1.255,
+      "theoretical_loss": 3.4694633837015125,
+      "tokens_seen": 1749090304
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004780606060606061,
+      "loss": 1.3452,
+      "theoretical_loss": 3.469452257354691,
+      "tokens_seen": 1749155840
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047804040404040405,
+      "loss": 1.4228,
+      "theoretical_loss": 3.469441131541454,
+      "tokens_seen": 1749221376
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000478020202020202,
+      "loss": 1.3645,
+      "theoretical_loss": 3.4694300062617565,
+      "tokens_seen": 1749286912
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047799999999999996,
+      "loss": 1.2568,
+      "theoretical_loss": 3.469418881515553,
+      "tokens_seen": 1749352448
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047797979797979803,
+      "loss": 1.2278,
+      "theoretical_loss": 3.4694077573027977,
+      "tokens_seen": 1749417984
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.000477959595959596,
+      "loss": 1.3498,
+      "theoretical_loss": 3.469396633623445,
+      "tokens_seen": 1749483520
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047793939393939394,
+      "loss": 1.201,
+      "theoretical_loss": 3.4693855104774496,
+      "tokens_seen": 1749549056
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004779191919191919,
+      "loss": 1.4381,
+      "theoretical_loss": 3.4693743878647654,
+      "tokens_seen": 1749614592
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004778989898989899,
+      "loss": 1.3039,
+      "theoretical_loss": 3.469363265785348,
+      "tokens_seen": 1749680128
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004778787878787879,
+      "loss": 1.6264,
+      "theoretical_loss": 3.46935214423915,
+      "tokens_seen": 1749745664
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004778585858585859,
+      "loss": 1.6328,
+      "theoretical_loss": 3.4693410232261277,
+      "tokens_seen": 1749811200
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047783838383838384,
+      "loss": 1.5476,
+      "theoretical_loss": 3.4693299027462343,
+      "tokens_seen": 1749876736
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004778181818181818,
+      "loss": 1.2902,
+      "theoretical_loss": 3.469318782799425,
+      "tokens_seen": 1749942272
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004777979797979798,
+      "loss": 1.4278,
+      "theoretical_loss": 3.4693076633856545,
+      "tokens_seen": 1750007808
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/aligned_token_count": 0.4995723366737366,
+      "objective/train/avg_token_score": 0.01636279560625553,
+      "objective/train/docs_used": 990903,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.7601079940795898,
+      "objective/train/original_loss": 4.128569602966309,
+      "objective/train/theoretical_loss": 3.4692965445048767,
+      "objective/train/tokens_used": 108932576,
+      "objective/train/unlikelihood_loss": 0.05954797938466072,
+      "theoretical_loss": 3.4692965445048767,
+      "tokens_seen": 1750073344
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004777777777777778,
+      "loss": 1.532,
+      "theoretical_loss": 3.4692965445048767,
+      "tokens_seen": 1750073344
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047775757575757577,
+      "loss": 1.3282,
+      "theoretical_loss": 3.469285426157046,
+      "tokens_seen": 1750138880
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047773737373737373,
+      "loss": 1.3803,
+      "theoretical_loss": 3.469274308342117,
+      "tokens_seen": 1750204416
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047771717171717174,
+      "loss": 1.5482,
+      "theoretical_loss": 3.469263191060045,
+      "tokens_seen": 1750269952
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004776969696969697,
+      "loss": 1.6532,
+      "theoretical_loss": 3.4692520743107833,
+      "tokens_seen": 1750335488
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004776767676767677,
+      "loss": 1.4167,
+      "theoretical_loss": 3.469240958094287,
+      "tokens_seen": 1750401024
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047765656565656566,
+      "loss": 1.193,
+      "theoretical_loss": 3.469229842410511,
+      "tokens_seen": 1750466560
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004776363636363636,
+      "loss": 1.3584,
+      "theoretical_loss": 3.4692187272594093,
+      "tokens_seen": 1750532096
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047761616161616163,
+      "loss": 1.439,
+      "theoretical_loss": 3.469207612640936,
+      "tokens_seen": 1750597632
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004775959595959596,
+      "loss": 1.2879,
+      "theoretical_loss": 3.469196498555047,
+      "tokens_seen": 1750663168
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004775757575757576,
+      "loss": 1.3754,
+      "theoretical_loss": 3.469185385001696,
+      "tokens_seen": 1750728704
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047755555555555556,
+      "loss": 1.2025,
+      "theoretical_loss": 3.4691742719808376,
+      "tokens_seen": 1750794240
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047753535353535357,
+      "loss": 1.5471,
+      "theoretical_loss": 3.469163159492426,
+      "tokens_seen": 1750859776
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004775151515151515,
+      "loss": 1.2861,
+      "theoretical_loss": 3.469152047536416,
+      "tokens_seen": 1750925312
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004774949494949495,
+      "loss": 1.4033,
+      "theoretical_loss": 3.4691409361127628,
+      "tokens_seen": 1750990848
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004774747474747475,
+      "loss": 1.3833,
+      "theoretical_loss": 3.46912982522142,
+      "tokens_seen": 1751056384
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047745454545454545,
+      "loss": 1.5292,
+      "theoretical_loss": 3.469118714862343,
+      "tokens_seen": 1751121920
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047743434343434346,
+      "loss": 1.3997,
+      "theoretical_loss": 3.469107605035486,
+      "tokens_seen": 1751187456
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004774141414141414,
+      "loss": 1.3918,
+      "theoretical_loss": 3.4690964957408035,
+      "tokens_seen": 1751252992
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004773939393939394,
+      "loss": 1.5265,
+      "theoretical_loss": 3.46908538697825,
+      "tokens_seen": 1751318528
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004773737373737374,
+      "loss": 1.2903,
+      "theoretical_loss": 3.4690742787477804,
+      "tokens_seen": 1751384064
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004773535353535354,
+      "loss": 1.4772,
+      "theoretical_loss": 3.4690631710493496,
+      "tokens_seen": 1751449600
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047733333333333335,
+      "loss": 1.4411,
+      "theoretical_loss": 3.469052063882912,
+      "tokens_seen": 1751515136
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.0004773131313131313,
+      "loss": 1.4693,
+      "theoretical_loss": 3.4690409572484215,
+      "tokens_seen": 1751580672
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 0.00047729292929292927,
+      "loss": 1.6307,
+      "theoretical_loss": 3.469029851145833,
+      "tokens_seen": 1751646208
+    },
+    {
+      "epoch": 0.05,
+      "objective/train/aligned_token_count": 0.35160067677497864,
+      "objective/train/avg_token_score": 0.0432390458881855,
+      "objective/train/docs_used": 991641,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.203386664390564,
+      "objective/train/original_loss": 4.336913108825684,
+      "objective/train/theoretical_loss": 3.469018745575102,
+      "objective/train/tokens_used": 110570976,
+      "objective/train/unlikelihood_loss": 0.058832619339227676,
+      "theoretical_loss": 3.469018745575102,
+      "tokens_seen": 1751711744
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004772727272727273,
+      "loss": 1.3743,
+      "theoretical_loss": 3.469018745575102,
+      "tokens_seen": 1751711744
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004772525252525253,
+      "loss": 1.4335,
+      "theoretical_loss": 3.469007640536182,
+      "tokens_seen": 1751777280
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047723232323232325,
+      "loss": 1.5358,
+      "theoretical_loss": 3.4689965360290285,
+      "tokens_seen": 1751842816
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004772121212121212,
+      "loss": 1.2956,
+      "theoretical_loss": 3.4689854320535956,
+      "tokens_seen": 1751908352
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047719191919191916,
+      "loss": 1.5891,
+      "theoretical_loss": 3.4689743286098382,
+      "tokens_seen": 1751973888
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004771717171717172,
+      "loss": 1.5675,
+      "theoretical_loss": 3.468963225697711,
+      "tokens_seen": 1752039424
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004771515151515152,
+      "loss": 1.3356,
+      "theoretical_loss": 3.4689521233171687,
+      "tokens_seen": 1752104960
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047713131313131314,
+      "loss": 1.4772,
+      "theoretical_loss": 3.468941021468166,
+      "tokens_seen": 1752170496
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004771111111111111,
+      "loss": 1.2844,
+      "theoretical_loss": 3.468929920150657,
+      "tokens_seen": 1752236032
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047709090909090905,
+      "loss": 1.5272,
+      "theoretical_loss": 3.468918819364597,
+      "tokens_seen": 1752301568
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004770707070707071,
+      "loss": 1.4227,
+      "theoretical_loss": 3.4689077191099402,
+      "tokens_seen": 1752367104
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004770505050505051,
+      "loss": 1.2629,
+      "theoretical_loss": 3.4688966193866415,
+      "tokens_seen": 1752432640
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047703030303030303,
+      "loss": 1.4331,
+      "theoretical_loss": 3.468885520194656,
+      "tokens_seen": 1752498176
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000477010101010101,
+      "loss": 1.4719,
+      "theoretical_loss": 3.468874421533938,
+      "tokens_seen": 1752563712
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000476989898989899,
+      "loss": 1.4844,
+      "theoretical_loss": 3.4688633234044426,
+      "tokens_seen": 1752629248
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000476969696969697,
+      "loss": 1.6083,
+      "theoretical_loss": 3.468852225806123,
+      "tokens_seen": 1752694784
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047694949494949497,
+      "loss": 1.3576,
+      "theoretical_loss": 3.4688411287389362,
+      "tokens_seen": 1752760320
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004769292929292929,
+      "loss": 1.5143,
+      "theoretical_loss": 3.4688300322028356,
+      "tokens_seen": 1752825856
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004769090909090909,
+      "loss": 1.4537,
+      "theoretical_loss": 3.468818936197776,
+      "tokens_seen": 1752891392
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004768888888888889,
+      "loss": 1.5708,
+      "theoretical_loss": 3.4688078407237124,
+      "tokens_seen": 1752956928
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004768686868686869,
+      "loss": 1.4414,
+      "theoretical_loss": 3.4687967457805993,
+      "tokens_seen": 1753022464
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047684848484848486,
+      "loss": 1.5843,
+      "theoretical_loss": 3.4687856513683917,
+      "tokens_seen": 1753088000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004768282828282828,
+      "loss": 1.5725,
+      "theoretical_loss": 3.468774557487044,
+      "tokens_seen": 1753153536
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047680808080808083,
+      "loss": 1.2515,
+      "theoretical_loss": 3.4687634641365115,
+      "tokens_seen": 1753219072
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004767878787878788,
+      "loss": 1.7292,
+      "theoretical_loss": 3.4687523713167483,
+      "tokens_seen": 1753284608
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/aligned_token_count": 0.5292032957077026,
+      "objective/train/avg_token_score": 0.022525779902935028,
+      "objective/train/docs_used": 992944,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.7439391613006592,
+      "objective/train/original_loss": 3.8954503536224365,
+      "objective/train/theoretical_loss": 3.468741279027709,
+      "objective/train/tokens_used": 112209376,
+      "objective/train/unlikelihood_loss": 0.06822893768548965,
+      "theoretical_loss": 3.468741279027709,
+      "tokens_seen": 1753350144
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004767676767676768,
+      "loss": 1.5733,
+      "theoretical_loss": 3.468741279027709,
+      "tokens_seen": 1753350144
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047674747474747475,
+      "loss": 1.3831,
+      "theoretical_loss": 3.4687301872693497,
+      "tokens_seen": 1753415680
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004767272727272727,
+      "loss": 1.2289,
+      "theoretical_loss": 3.4687190960416245,
+      "tokens_seen": 1753481216
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004767070707070707,
+      "loss": 1.2174,
+      "theoretical_loss": 3.468708005344487,
+      "tokens_seen": 1753546752
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047668686868686873,
+      "loss": 1.4894,
+      "theoretical_loss": 3.468696915177894,
+      "tokens_seen": 1753612288
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004766666666666667,
+      "loss": 1.7117,
+      "theoretical_loss": 3.4686858255417987,
+      "tokens_seen": 1753677824
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047664646464646465,
+      "loss": 1.5382,
+      "theoretical_loss": 3.4686747364361565,
+      "tokens_seen": 1753743360
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047662626262626266,
+      "loss": 1.6347,
+      "theoretical_loss": 3.4686636478609225,
+      "tokens_seen": 1753808896
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004766060606060606,
+      "loss": 1.3016,
+      "theoretical_loss": 3.4686525598160514,
+      "tokens_seen": 1753874432
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004765858585858586,
+      "loss": 1.2402,
+      "theoretical_loss": 3.4686414723014973,
+      "tokens_seen": 1753939968
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004765656565656566,
+      "loss": 1.6342,
+      "theoretical_loss": 3.4686303853172165,
+      "tokens_seen": 1754005504
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047654545454545454,
+      "loss": 1.4743,
+      "theoretical_loss": 3.468619298863162,
+      "tokens_seen": 1754071040
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047652525252525255,
+      "loss": 1.5816,
+      "theoretical_loss": 3.46860821293929,
+      "tokens_seen": 1754136576
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004765050505050505,
+      "loss": 1.3497,
+      "theoretical_loss": 3.4685971275455545,
+      "tokens_seen": 1754202112
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004764848484848485,
+      "loss": 1.4686,
+      "theoretical_loss": 3.4685860426819106,
+      "tokens_seen": 1754267648
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004764646464646465,
+      "loss": 1.5528,
+      "theoretical_loss": 3.468574958348314,
+      "tokens_seen": 1754333184
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004764444444444445,
+      "loss": 1.4893,
+      "theoretical_loss": 3.468563874544718,
+      "tokens_seen": 1754398720
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047642424242424244,
+      "loss": 1.5118,
+      "theoretical_loss": 3.4685527912710787,
+      "tokens_seen": 1754464256
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004764040404040404,
+      "loss": 1.5134,
+      "theoretical_loss": 3.4685417085273507,
+      "tokens_seen": 1754529792
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004763838383838384,
+      "loss": 1.6654,
+      "theoretical_loss": 3.468530626313488,
+      "tokens_seen": 1754595328
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047636363636363637,
+      "loss": 1.5252,
+      "theoretical_loss": 3.468519544629447,
+      "tokens_seen": 1754660864
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004763434343434344,
+      "loss": 1.2085,
+      "theoretical_loss": 3.468508463475181,
+      "tokens_seen": 1754726400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047632323232323233,
+      "loss": 1.4196,
+      "theoretical_loss": 3.4684973828506465,
+      "tokens_seen": 1754791936
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004763030303030303,
+      "loss": 1.6456,
+      "theoretical_loss": 3.468486302755797,
+      "tokens_seen": 1754857472
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004762828282828283,
+      "loss": 1.5107,
+      "theoretical_loss": 3.4684752231905875,
+      "tokens_seen": 1754923008
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/aligned_token_count": 0.3735337257385254,
+      "objective/train/avg_token_score": 0.013170573860406876,
+      "objective/train/docs_used": 993452,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.3061128854751587,
+      "objective/train/original_loss": 4.716046333312988,
+      "objective/train/theoretical_loss": 3.468464144154974,
+      "objective/train/tokens_used": 113847776,
+      "objective/train/unlikelihood_loss": 0.05805331468582153,
+      "theoretical_loss": 3.468464144154974,
+      "tokens_seen": 1754988544
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004762626262626263,
+      "loss": 1.1821,
+      "theoretical_loss": 3.468464144154974,
+      "tokens_seen": 1754988544
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047624242424242427,
+      "loss": 1.3812,
+      "theoretical_loss": 3.468453065648911,
+      "tokens_seen": 1755054080
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004762222222222222,
+      "loss": 1.5309,
+      "theoretical_loss": 3.468441987672353,
+      "tokens_seen": 1755119616
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004762020202020202,
+      "loss": 1.5255,
+      "theoretical_loss": 3.468430910225255,
+      "tokens_seen": 1755185152
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004761818181818182,
+      "loss": 1.4808,
+      "theoretical_loss": 3.468419833307572,
+      "tokens_seen": 1755250688
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004761616161616162,
+      "loss": 1.5992,
+      "theoretical_loss": 3.468408756919259,
+      "tokens_seen": 1755316224
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047614141414141416,
+      "loss": 1.4779,
+      "theoretical_loss": 3.468397681060271,
+      "tokens_seen": 1755381760
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004761212121212121,
+      "loss": 1.4515,
+      "theoretical_loss": 3.4683866057305623,
+      "tokens_seen": 1755447296
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004761010101010101,
+      "loss": 1.3284,
+      "theoretical_loss": 3.468375530930089,
+      "tokens_seen": 1755512832
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004760808080808081,
+      "loss": 1.4936,
+      "theoretical_loss": 3.468364456658805,
+      "tokens_seen": 1755578368
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004760606060606061,
+      "loss": 1.4289,
+      "theoretical_loss": 3.4683533829166664,
+      "tokens_seen": 1755643904
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047604040404040406,
+      "loss": 1.5279,
+      "theoretical_loss": 3.468342309703627,
+      "tokens_seen": 1755709440
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000476020202020202,
+      "loss": 1.1404,
+      "theoretical_loss": 3.4683312370196426,
+      "tokens_seen": 1755774976
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047599999999999997,
+      "loss": 1.6458,
+      "theoretical_loss": 3.4683201648646675,
+      "tokens_seen": 1755840512
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047597979797979803,
+      "loss": 1.3894,
+      "theoretical_loss": 3.4683090932386573,
+      "tokens_seen": 1755906048
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000475959595959596,
+      "loss": 1.5657,
+      "theoretical_loss": 3.4682980221415662,
+      "tokens_seen": 1755971584
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047593939393939395,
+      "loss": 1.2969,
+      "theoretical_loss": 3.46828695157335,
+      "tokens_seen": 1756037120
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004759191919191919,
+      "loss": 1.4629,
+      "theoretical_loss": 3.4682758815339634,
+      "tokens_seen": 1756102656
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047589898989898986,
+      "loss": 1.4144,
+      "theoretical_loss": 3.4682648120233615,
+      "tokens_seen": 1756168192
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047587878787878793,
+      "loss": 1.2853,
+      "theoretical_loss": 3.4682537430414992,
+      "tokens_seen": 1756233728
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004758585858585859,
+      "loss": 1.1362,
+      "theoretical_loss": 3.468242674588331,
+      "tokens_seen": 1756299264
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047583838383838384,
+      "loss": 1.4003,
+      "theoretical_loss": 3.468231606663813,
+      "tokens_seen": 1756364800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004758181818181818,
+      "loss": 1.3655,
+      "theoretical_loss": 3.4682205392678993,
+      "tokens_seen": 1756430336
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004757979797979798,
+      "loss": 1.2837,
+      "theoretical_loss": 3.4682094724005457,
+      "tokens_seen": 1756495872
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004757777777777778,
+      "loss": 1.2286,
+      "theoretical_loss": 3.4681984060617066,
+      "tokens_seen": 1756561408
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/aligned_token_count": 0.4603494703769684,
+      "objective/train/avg_token_score": 0.007801194675266743,
+      "objective/train/docs_used": 994709,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.4813201427459717,
+      "objective/train/original_loss": 3.937505006790161,
+      "objective/train/theoretical_loss": 3.4681873402513372,
+      "objective/train/tokens_used": 115486176,
+      "objective/train/unlikelihood_loss": 0.07865725457668304,
+      "theoretical_loss": 3.4681873402513372,
+      "tokens_seen": 1756626944
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004757575757575758,
+      "loss": 1.5432,
+      "theoretical_loss": 3.4681873402513372,
+      "tokens_seen": 1756626944
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047573737373737373,
+      "loss": 1.328,
+      "theoretical_loss": 3.4681762749693927,
+      "tokens_seen": 1756692480
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004757171717171717,
+      "loss": 1.3619,
+      "theoretical_loss": 3.4681652102158282,
+      "tokens_seen": 1756758016
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004756969696969697,
+      "loss": 1.2638,
+      "theoretical_loss": 3.4681541459905985,
+      "tokens_seen": 1756823552
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004756767676767677,
+      "loss": 1.5705,
+      "theoretical_loss": 3.468143082293659,
+      "tokens_seen": 1756889088
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047565656565656567,
+      "loss": 1.3297,
+      "theoretical_loss": 3.4681320191249645,
+      "tokens_seen": 1756954624
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004756363636363636,
+      "loss": 1.2614,
+      "theoretical_loss": 3.46812095648447,
+      "tokens_seen": 1757020160
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047561616161616164,
+      "loss": 1.3043,
+      "theoretical_loss": 3.468109894372131,
+      "tokens_seen": 1757085696
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004755959595959596,
+      "loss": 1.4471,
+      "theoretical_loss": 3.468098832787902,
+      "tokens_seen": 1757151232
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004755757575757576,
+      "loss": 1.4198,
+      "theoretical_loss": 3.4680877717317387,
+      "tokens_seen": 1757216768
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047555555555555556,
+      "loss": 1.4285,
+      "theoretical_loss": 3.4680767112035955,
+      "tokens_seen": 1757282304
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004755353535353535,
+      "loss": 1.3098,
+      "theoretical_loss": 3.4680656512034282,
+      "tokens_seen": 1757347840
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047551515151515153,
+      "loss": 1.4135,
+      "theoretical_loss": 3.4680545917311916,
+      "tokens_seen": 1757413376
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004754949494949495,
+      "loss": 1.2951,
+      "theoretical_loss": 3.468043532786841,
+      "tokens_seen": 1757478912
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004754747474747475,
+      "loss": 1.4952,
+      "theoretical_loss": 3.4680324743703315,
+      "tokens_seen": 1757544448
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047545454545454545,
+      "loss": 1.3478,
+      "theoretical_loss": 3.4680214164816174,
+      "tokens_seen": 1757609984
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047543434343434347,
+      "loss": 1.4044,
+      "theoretical_loss": 3.468010359120655,
+      "tokens_seen": 1757675520
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004754141414141414,
+      "loss": 1.3726,
+      "theoretical_loss": 3.467999302287399,
+      "tokens_seen": 1757741056
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004753939393939394,
+      "loss": 1.5165,
+      "theoretical_loss": 3.4679882459818043,
+      "tokens_seen": 1757806592
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004753737373737374,
+      "loss": 1.6779,
+      "theoretical_loss": 3.4679771902038263,
+      "tokens_seen": 1757872128
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047535353535353535,
+      "loss": 1.3127,
+      "theoretical_loss": 3.4679661349534197,
+      "tokens_seen": 1757937664
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047533333333333336,
+      "loss": 1.4477,
+      "theoretical_loss": 3.4679550802305408,
+      "tokens_seen": 1758003200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004753131313131313,
+      "loss": 1.3832,
+      "theoretical_loss": 3.467944026035144,
+      "tokens_seen": 1758068736
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047529292929292927,
+      "loss": 1.3038,
+      "theoretical_loss": 3.4679329723671835,
+      "tokens_seen": 1758134272
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004752727272727273,
+      "loss": 1.3008,
+      "theoretical_loss": 3.467921919226616,
+      "tokens_seen": 1758199808
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/aligned_token_count": 0.37341153621673584,
+      "objective/train/avg_token_score": 0.01459649857133627,
+      "objective/train/docs_used": 995465,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.182531476020813,
+      "objective/train/original_loss": 4.433279991149902,
+      "objective/train/theoretical_loss": 3.4679108666133964,
+      "objective/train/tokens_used": 117124576,
+      "objective/train/unlikelihood_loss": 0.05860118195414543,
+      "theoretical_loss": 3.4679108666133964,
+      "tokens_seen": 1758265344
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004752525252525253,
+      "loss": 1.3201,
+      "theoretical_loss": 3.4679108666133964,
+      "tokens_seen": 1758265344
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047523232323232325,
+      "loss": 1.4253,
+      "theoretical_loss": 3.4678998145274793,
+      "tokens_seen": 1758330880
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004752121212121212,
+      "loss": 1.2983,
+      "theoretical_loss": 3.467888762968821,
+      "tokens_seen": 1758396416
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047519191919191916,
+      "loss": 1.4404,
+      "theoretical_loss": 3.467877711937375,
+      "tokens_seen": 1758461952
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004751717171717172,
+      "loss": 1.4583,
+      "theoretical_loss": 3.4678666614330975,
+      "tokens_seen": 1758527488
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004751515151515152,
+      "loss": 1.4141,
+      "theoretical_loss": 3.467855611455944,
+      "tokens_seen": 1758593024
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047513131313131314,
+      "loss": 1.527,
+      "theoretical_loss": 3.467844562005869,
+      "tokens_seen": 1758658560
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004751111111111111,
+      "loss": 1.3646,
+      "theoretical_loss": 3.4678335130828284,
+      "tokens_seen": 1758724096
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047509090909090906,
+      "loss": 1.2391,
+      "theoretical_loss": 3.4678224646867766,
+      "tokens_seen": 1758789632
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004750707070707071,
+      "loss": 1.281,
+      "theoretical_loss": 3.4678114168176695,
+      "tokens_seen": 1758855168
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004750505050505051,
+      "loss": 1.392,
+      "theoretical_loss": 3.467800369475462,
+      "tokens_seen": 1758920704
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047503030303030304,
+      "loss": 1.2578,
+      "theoretical_loss": 3.46778932266011,
+      "tokens_seen": 1758986240
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000475010101010101,
+      "loss": 1.4468,
+      "theoretical_loss": 3.467778276371568,
+      "tokens_seen": 1759051776
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000474989898989899,
+      "loss": 1.3945,
+      "theoretical_loss": 3.467767230609791,
+      "tokens_seen": 1759117312
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000474969696969697,
+      "loss": 1.166,
+      "theoretical_loss": 3.4677561853747356,
+      "tokens_seen": 1759182848
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047494949494949497,
+      "loss": 1.5393,
+      "theoretical_loss": 3.4677451406663558,
+      "tokens_seen": 1759248384
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047492929292929293,
+      "loss": 1.6032,
+      "theoretical_loss": 3.467734096484607,
+      "tokens_seen": 1759313920
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004749090909090909,
+      "loss": 1.4508,
+      "theoretical_loss": 3.467723052829445,
+      "tokens_seen": 1759379456
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047488888888888895,
+      "loss": 1.5765,
+      "theoretical_loss": 3.4677120097008247,
+      "tokens_seen": 1759444992
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004748686868686869,
+      "loss": 1.3183,
+      "theoretical_loss": 3.4677009670987013,
+      "tokens_seen": 1759510528
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047484848484848487,
+      "loss": 1.3187,
+      "theoretical_loss": 3.4676899250230306,
+      "tokens_seen": 1759576064
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004748282828282828,
+      "loss": 1.3925,
+      "theoretical_loss": 3.4676788834737677,
+      "tokens_seen": 1759641600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004748080808080808,
+      "loss": 1.4587,
+      "theoretical_loss": 3.467667842450868,
+      "tokens_seen": 1759707136
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047478787878787884,
+      "loss": 1.4543,
+      "theoretical_loss": 3.467656801954286,
+      "tokens_seen": 1759772672
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004747676767676768,
+      "loss": 1.3954,
+      "theoretical_loss": 3.4676457619839773,
+      "tokens_seen": 1759838208
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/aligned_token_count": 0.3027859330177307,
+      "objective/train/avg_token_score": 0.026031043380498886,
+      "objective/train/docs_used": 996601,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.0335179567337036,
+      "objective/train/original_loss": 4.650774955749512,
+      "objective/train/theoretical_loss": 3.4676347225398985,
+      "objective/train/tokens_used": 118762976,
+      "objective/train/unlikelihood_loss": 0.07695260643959045,
+      "theoretical_loss": 3.4676347225398985,
+      "tokens_seen": 1759903744
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047474747474747476,
+      "loss": 1.1674,
+      "theoretical_loss": 3.4676347225398985,
+      "tokens_seen": 1759903744
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004747272727272727,
+      "loss": 1.283,
+      "theoretical_loss": 3.467623683622003,
+      "tokens_seen": 1759969280
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047470707070707067,
+      "loss": 1.4078,
+      "theoretical_loss": 3.467612645230248,
+      "tokens_seen": 1760034816
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047468686868686874,
+      "loss": 1.3631,
+      "theoretical_loss": 3.4676016073645872,
+      "tokens_seen": 1760100352
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004746666666666667,
+      "loss": 1.4091,
+      "theoretical_loss": 3.467590570024977,
+      "tokens_seen": 1760165888
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047464646464646465,
+      "loss": 1.4418,
+      "theoretical_loss": 3.4675795332113726,
+      "tokens_seen": 1760231424
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004746262626262626,
+      "loss": 1.3409,
+      "theoretical_loss": 3.467568496923729,
+      "tokens_seen": 1760296960
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004746060606060606,
+      "loss": 1.518,
+      "theoretical_loss": 3.4675574611620013,
+      "tokens_seen": 1760362496
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047458585858585863,
+      "loss": 1.4428,
+      "theoretical_loss": 3.4675464259261455,
+      "tokens_seen": 1760428032
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004745656565656566,
+      "loss": 1.3315,
+      "theoretical_loss": 3.467535391216116,
+      "tokens_seen": 1760493568
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047454545454545454,
+      "loss": 1.5314,
+      "theoretical_loss": 3.46752435703187,
+      "tokens_seen": 1760559104
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004745252525252525,
+      "loss": 1.5786,
+      "theoretical_loss": 3.4675133233733613,
+      "tokens_seen": 1760624640
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004745050505050505,
+      "loss": 1.5052,
+      "theoretical_loss": 3.467502290240546,
+      "tokens_seen": 1760690176
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004744848484848485,
+      "loss": 1.5072,
+      "theoretical_loss": 3.4674912576333785,
+      "tokens_seen": 1760755712
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004744646464646465,
+      "loss": 1.3799,
+      "theoretical_loss": 3.4674802255518156,
+      "tokens_seen": 1760821248
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047444444444444444,
+      "loss": 1.359,
+      "theoretical_loss": 3.467469193995812,
+      "tokens_seen": 1760886784
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047442424242424245,
+      "loss": 1.5932,
+      "theoretical_loss": 3.4674581629653227,
+      "tokens_seen": 1760952320
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004744040404040404,
+      "loss": 1.2436,
+      "theoretical_loss": 3.467447132460304,
+      "tokens_seen": 1761017856
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004743838383838384,
+      "loss": 1.2935,
+      "theoretical_loss": 3.467436102480711,
+      "tokens_seen": 1761083392
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047436363636363637,
+      "loss": 1.6615,
+      "theoretical_loss": 3.4674250730264986,
+      "tokens_seen": 1761148928
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047434343434343433,
+      "loss": 1.4143,
+      "theoretical_loss": 3.4674140440976227,
+      "tokens_seen": 1761214464
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047432323232323234,
+      "loss": 1.5773,
+      "theoretical_loss": 3.467403015694039,
+      "tokens_seen": 1761280000
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004743030303030303,
+      "loss": 1.4823,
+      "theoretical_loss": 3.467391987815702,
+      "tokens_seen": 1761345536
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004742828282828283,
+      "loss": 1.282,
+      "theoretical_loss": 3.4673809604625676,
+      "tokens_seen": 1761411072
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047426262626262626,
+      "loss": 1.5621,
+      "theoretical_loss": 3.4673699336345916,
+      "tokens_seen": 1761476608
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/aligned_token_count": 0.4690249264240265,
+      "objective/train/avg_token_score": 0.015736954286694527,
+      "objective/train/docs_used": 997243,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.6199918985366821,
+      "objective/train/original_loss": 3.9470674991607666,
+      "objective/train/theoretical_loss": 3.4673589073317297,
+      "objective/train/tokens_used": 120401376,
+      "objective/train/unlikelihood_loss": 0.07392548024654388,
+      "theoretical_loss": 3.4673589073317297,
+      "tokens_seen": 1761542144
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004742424242424243,
+      "loss": 1.4835,
+      "theoretical_loss": 3.4673589073317297,
+      "tokens_seen": 1761542144
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047422222222222223,
+      "loss": 1.4607,
+      "theoretical_loss": 3.4673478815539363,
+      "tokens_seen": 1761607680
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004742020202020202,
+      "loss": 1.655,
+      "theoretical_loss": 3.4673368563011677,
+      "tokens_seen": 1761673216
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004741818181818182,
+      "loss": 1.4306,
+      "theoretical_loss": 3.467325831573379,
+      "tokens_seen": 1761738752
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047416161616161616,
+      "loss": 1.4634,
+      "theoretical_loss": 3.467314807370526,
+      "tokens_seen": 1761804288
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047414141414141417,
+      "loss": 1.4165,
+      "theoretical_loss": 3.4673037836925635,
+      "tokens_seen": 1761869824
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004741212121212121,
+      "loss": 1.4309,
+      "theoretical_loss": 3.467292760539448,
+      "tokens_seen": 1761935360
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004741010101010101,
+      "loss": 1.3544,
+      "theoretical_loss": 3.467281737911134,
+      "tokens_seen": 1762000896
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004740808080808081,
+      "loss": 1.4597,
+      "theoretical_loss": 3.4672707158075777,
+      "tokens_seen": 1762066432
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004740606060606061,
+      "loss": 1.5272,
+      "theoretical_loss": 3.467259694228734,
+      "tokens_seen": 1762131968
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047404040404040406,
+      "loss": 1.3645,
+      "theoretical_loss": 3.467248673174559,
+      "tokens_seen": 1762197504
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000474020202020202,
+      "loss": 1.2698,
+      "theoretical_loss": 3.467237652645008,
+      "tokens_seen": 1762263040
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000474,
+      "loss": 1.1518,
+      "theoretical_loss": 3.4672266326400365,
+      "tokens_seen": 1762328576
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000473979797979798,
+      "loss": 1.446,
+      "theoretical_loss": 3.4672156131595995,
+      "tokens_seen": 1762394112
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000473959595959596,
+      "loss": 1.4405,
+      "theoretical_loss": 3.4672045942036536,
+      "tokens_seen": 1762459648
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047393939393939395,
+      "loss": 1.2889,
+      "theoretical_loss": 3.4671935757721535,
+      "tokens_seen": 1762525184
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004739191919191919,
+      "loss": 1.389,
+      "theoretical_loss": 3.467182557865055,
+      "tokens_seen": 1762590720
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047389898989898987,
+      "loss": 1.4165,
+      "theoretical_loss": 3.4671715404823136,
+      "tokens_seen": 1762656256
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047387878787878793,
+      "loss": 1.5323,
+      "theoretical_loss": 3.4671605236238854,
+      "tokens_seen": 1762721792
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004738585858585859,
+      "loss": 1.4547,
+      "theoretical_loss": 3.467149507289725,
+      "tokens_seen": 1762787328
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047383838383838385,
+      "loss": 1.3971,
+      "theoretical_loss": 3.4671384914797887,
+      "tokens_seen": 1762852864
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004738181818181818,
+      "loss": 1.4061,
+      "theoretical_loss": 3.4671274761940314,
+      "tokens_seen": 1762918400
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047379797979797976,
+      "loss": 1.2906,
+      "theoretical_loss": 3.4671164614324095,
+      "tokens_seen": 1762983936
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004737777777777778,
+      "loss": 1.4725,
+      "theoretical_loss": 3.4671054471948777,
+      "tokens_seen": 1763049472
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004737575757575758,
+      "loss": 1.2618,
+      "theoretical_loss": 3.467094433481392,
+      "tokens_seen": 1763115008
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/aligned_token_count": 0.3298509418964386,
+      "objective/train/avg_token_score": 0.013608359731733799,
+      "objective/train/docs_used": 998400,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.140276312828064,
+      "objective/train/original_loss": 4.645390033721924,
+      "objective/train/theoretical_loss": 3.4670834202919085,
+      "objective/train/tokens_used": 122039776,
+      "objective/train/unlikelihood_loss": 0.06240946799516678,
+      "theoretical_loss": 3.4670834202919085,
+      "tokens_seen": 1763180544
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047373737373737374,
+      "loss": 1.3174,
+      "theoretical_loss": 3.4670834202919085,
+      "tokens_seen": 1763180544
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004737171717171717,
+      "loss": 1.6851,
+      "theoretical_loss": 3.4670724076263815,
+      "tokens_seen": 1763246080
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004736969696969697,
+      "loss": 1.3992,
+      "theoretical_loss": 3.4670613954847678,
+      "tokens_seen": 1763311616
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004736767676767677,
+      "loss": 1.482,
+      "theoretical_loss": 3.467050383867023,
+      "tokens_seen": 1763377152
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004736565656565657,
+      "loss": 1.4294,
+      "theoretical_loss": 3.467039372773102,
+      "tokens_seen": 1763442688
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047363636363636363,
+      "loss": 1.4823,
+      "theoretical_loss": 3.467028362202961,
+      "tokens_seen": 1763508224
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004736161616161616,
+      "loss": 1.4347,
+      "theoretical_loss": 3.467017352156555,
+      "tokens_seen": 1763573760
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004735959595959596,
+      "loss": 1.5324,
+      "theoretical_loss": 3.46700634263384,
+      "tokens_seen": 1763639296
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004735757575757576,
+      "loss": 1.2494,
+      "theoretical_loss": 3.466995333634772,
+      "tokens_seen": 1763704832
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047355555555555557,
+      "loss": 1.3624,
+      "theoretical_loss": 3.4669843251593058,
+      "tokens_seen": 1763770368
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004735353535353535,
+      "loss": 1.3935,
+      "theoretical_loss": 3.4669733172073975,
+      "tokens_seen": 1763835904
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047351515151515154,
+      "loss": 1.4901,
+      "theoretical_loss": 3.466962309779003,
+      "tokens_seen": 1763901440
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004734949494949495,
+      "loss": 1.3198,
+      "theoretical_loss": 3.4669513028740777,
+      "tokens_seen": 1763966976
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004734747474747475,
+      "loss": 1.2222,
+      "theoretical_loss": 3.466940296492577,
+      "tokens_seen": 1764032512
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047345454545454546,
+      "loss": 1.4412,
+      "theoretical_loss": 3.466929290634457,
+      "tokens_seen": 1764098048
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004734343434343434,
+      "loss": 1.2884,
+      "theoretical_loss": 3.466918285299673,
+      "tokens_seen": 1764163584
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047341414141414143,
+      "loss": 1.6086,
+      "theoretical_loss": 3.466907280488181,
+      "tokens_seen": 1764229120
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004733939393939394,
+      "loss": 1.3749,
+      "theoretical_loss": 3.4668962761999365,
+      "tokens_seen": 1764294656
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004733737373737374,
+      "loss": 1.2535,
+      "theoretical_loss": 3.4668852724348955,
+      "tokens_seen": 1764360192
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047335353535353535,
+      "loss": 1.3333,
+      "theoretical_loss": 3.4668742691930126,
+      "tokens_seen": 1764425728
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047333333333333336,
+      "loss": 1.2971,
+      "theoretical_loss": 3.4668632664742454,
+      "tokens_seen": 1764491264
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004733131313131313,
+      "loss": 1.3483,
+      "theoretical_loss": 3.4668522642785478,
+      "tokens_seen": 1764556800
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004732929292929293,
+      "loss": 1.2886,
+      "theoretical_loss": 3.466841262605876,
+      "tokens_seen": 1764622336
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004732727272727273,
+      "loss": 1.4083,
+      "theoretical_loss": 3.466830261456187,
+      "tokens_seen": 1764687872
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047325252525252525,
+      "loss": 1.4522,
+      "theoretical_loss": 3.4668192608294346,
+      "tokens_seen": 1764753408
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/aligned_token_count": 0.30015885829925537,
+      "objective/train/avg_token_score": 0.025842411443591118,
+      "objective/train/docs_used": 998902,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 0.9383648633956909,
+      "objective/train/original_loss": 4.241371154785156,
+      "objective/train/theoretical_loss": 3.4668082607255757,
+      "objective/train/tokens_used": 123678176,
+      "objective/train/unlikelihood_loss": 0.10302804410457611,
+      "theoretical_loss": 3.4668082607255757,
+      "tokens_seen": 1764818944
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047323232323232326,
+      "loss": 1.4569,
+      "theoretical_loss": 3.4668082607255757,
+      "tokens_seen": 1764818944
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004732121212121212,
+      "loss": 1.4089,
+      "theoretical_loss": 3.4667972611445657,
+      "tokens_seen": 1764884480
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004731919191919192,
+      "loss": 1.4655,
+      "theoretical_loss": 3.46678626208636,
+      "tokens_seen": 1764950016
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004731717171717172,
+      "loss": 1.4081,
+      "theoretical_loss": 3.466775263550915,
+      "tokens_seen": 1765015552
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004731515151515152,
+      "loss": 1.5019,
+      "theoretical_loss": 3.466764265538186,
+      "tokens_seen": 1765081088
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047313131313131315,
+      "loss": 1.4618,
+      "theoretical_loss": 3.466753268048129,
+      "tokens_seen": 1765146624
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004731111111111111,
+      "loss": 1.2713,
+      "theoretical_loss": 3.4667422710806997,
+      "tokens_seen": 1765212160
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004730909090909091,
+      "loss": 1.2932,
+      "theoretical_loss": 3.466731274635854,
+      "tokens_seen": 1765277696
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004730707070707071,
+      "loss": 1.6351,
+      "theoretical_loss": 3.466720278713547,
+      "tokens_seen": 1765343232
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004730505050505051,
+      "loss": 1.5274,
+      "theoretical_loss": 3.466709283313735,
+      "tokens_seen": 1765408768
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047303030303030304,
+      "loss": 1.4793,
+      "theoretical_loss": 3.4666982884363735,
+      "tokens_seen": 1765474304
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000473010101010101,
+      "loss": 1.4963,
+      "theoretical_loss": 3.466687294081419,
+      "tokens_seen": 1765539840
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000472989898989899,
+      "loss": 1.3953,
+      "theoretical_loss": 3.466676300248827,
+      "tokens_seen": 1765605376
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000472969696969697,
+      "loss": 1.3848,
+      "theoretical_loss": 3.4666653069385527,
+      "tokens_seen": 1765670912
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.000472949494949495,
+      "loss": 1.3043,
+      "theoretical_loss": 3.466654314150552,
+      "tokens_seen": 1765736448
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047292929292929293,
+      "loss": 1.3659,
+      "theoretical_loss": 3.4666433218847814,
+      "tokens_seen": 1765801984
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004729090909090909,
+      "loss": 1.2315,
+      "theoretical_loss": 3.466632330141196,
+      "tokens_seen": 1765867520
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004728888888888889,
+      "loss": 1.412,
+      "theoretical_loss": 3.4666213389197518,
+      "tokens_seen": 1765933056
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004728686868686869,
+      "loss": 1.527,
+      "theoretical_loss": 3.4666103482204056,
+      "tokens_seen": 1765998592
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047284848484848487,
+      "loss": 1.1406,
+      "theoretical_loss": 3.466599358043111,
+      "tokens_seen": 1766064128
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047282828282828283,
+      "loss": 1.4347,
+      "theoretical_loss": 3.4665883683878262,
+      "tokens_seen": 1766129664
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004728080808080808,
+      "loss": 1.2191,
+      "theoretical_loss": 3.466577379254506,
+      "tokens_seen": 1766195200
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004727878787878788,
+      "loss": 1.3334,
+      "theoretical_loss": 3.4665663906431057,
+      "tokens_seen": 1766260736
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004727676767676768,
+      "loss": 1.3408,
+      "theoretical_loss": 3.466555402553582,
+      "tokens_seen": 1766326272
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047274747474747476,
+      "loss": 1.424,
+      "theoretical_loss": 3.466544414985891,
+      "tokens_seen": 1766391808
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/aligned_token_count": 0.5058650970458984,
+      "objective/train/avg_token_score": 0.020609397441148758,
+      "objective/train/docs_used": 999561,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.6941487789154053,
+      "objective/train/original_loss": 3.954047918319702,
+      "objective/train/theoretical_loss": 3.4665334279399875,
+      "objective/train/tokens_used": 125316576,
+      "objective/train/unlikelihood_loss": 0.04431372508406639,
+      "theoretical_loss": 3.4665334279399875,
+      "tokens_seen": 1766457344
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004727272727272727,
+      "loss": 1.5081,
+      "theoretical_loss": 3.4665334279399875,
+      "tokens_seen": 1766457344
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004727070707070707,
+      "loss": 1.2911,
+      "theoretical_loss": 3.4665224414158278,
+      "tokens_seen": 1766522880
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047268686868686874,
+      "loss": 1.4671,
+      "theoretical_loss": 3.466511455413368,
+      "tokens_seen": 1766588416
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004726666666666667,
+      "loss": 1.3627,
+      "theoretical_loss": 3.466500469932564,
+      "tokens_seen": 1766653952
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047264646464646466,
+      "loss": 1.6863,
+      "theoretical_loss": 3.466489484973371,
+      "tokens_seen": 1766719488
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004726262626262626,
+      "loss": 1.3115,
+      "theoretical_loss": 3.4664785005357457,
+      "tokens_seen": 1766785024
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047260606060606057,
+      "loss": 1.4088,
+      "theoretical_loss": 3.4664675166196437,
+      "tokens_seen": 1766850560
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047258585858585864,
+      "loss": 1.2904,
+      "theoretical_loss": 3.4664565332250215,
+      "tokens_seen": 1766916096
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004725656565656566,
+      "loss": 1.5834,
+      "theoretical_loss": 3.4664455503518337,
+      "tokens_seen": 1766981632
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047254545454545455,
+      "loss": 1.5353,
+      "theoretical_loss": 3.466434568000037,
+      "tokens_seen": 1767047168
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004725252525252525,
+      "loss": 1.2129,
+      "theoretical_loss": 3.4664235861695873,
+      "tokens_seen": 1767112704
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004725050505050505,
+      "loss": 1.4923,
+      "theoretical_loss": 3.46641260486044,
+      "tokens_seen": 1767178240
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047248484848484853,
+      "loss": 1.6744,
+      "theoretical_loss": 3.466401624072552,
+      "tokens_seen": 1767243776
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004724646464646465,
+      "loss": 1.1128,
+      "theoretical_loss": 3.4663906438058785,
+      "tokens_seen": 1767309312
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047244444444444444,
+      "loss": 1.5299,
+      "theoretical_loss": 3.4663796640603755,
+      "tokens_seen": 1767374848
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004724242424242424,
+      "loss": 1.4135,
+      "theoretical_loss": 3.466368684835999,
+      "tokens_seen": 1767440384
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004724040404040404,
+      "loss": 1.3394,
+      "theoretical_loss": 3.466357706132705,
+      "tokens_seen": 1767505920
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004723838383838384,
+      "loss": 1.3015,
+      "theoretical_loss": 3.46634672795045,
+      "tokens_seen": 1767571456
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004723636363636364,
+      "loss": 1.3472,
+      "theoretical_loss": 3.466335750289189,
+      "tokens_seen": 1767636992
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047234343434343433,
+      "loss": 1.2866,
+      "theoretical_loss": 3.466324773148878,
+      "tokens_seen": 1767702528
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047232323232323235,
+      "loss": 1.6653,
+      "theoretical_loss": 3.466313796529474,
+      "tokens_seen": 1767768064
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004723030303030303,
+      "loss": 1.5335,
+      "theoretical_loss": 3.466302820430932,
+      "tokens_seen": 1767833600
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004722828282828283,
+      "loss": 1.5068,
+      "theoretical_loss": 3.466291844853208,
+      "tokens_seen": 1767899136
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00047226262626262627,
+      "loss": 1.3592,
+      "theoretical_loss": 3.4662808697962584,
+      "tokens_seen": 1767964672
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.0004722424242424242,
+      "loss": 1.4486,
+      "theoretical_loss": 3.466269895260039,
+      "tokens_seen": 1768030208
+    },
+    {
+      "epoch": 0.06,
+      "objective/train/aligned_token_count": 0.3491568863391876,
+      "objective/train/avg_token_score": 0.024049675092101097,
+      "objective/train/docs_used": 1000589,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.332099437713623,
+      "objective/train/original_loss": 4.583323001861572,
+      "objective/train/theoretical_loss": 3.466258921244506,
+      "objective/train/tokens_used": 126954976,
+      "objective/train/unlikelihood_loss": 0.07067796587944031,
+      "theoretical_loss": 3.466258921244506,
+      "tokens_seen": 1768095744
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047222222222222224,
+      "loss": 1.5076,
+      "theoretical_loss": 3.466258921244506,
+      "tokens_seen": 1768095744
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004722020202020202,
+      "loss": 1.2966,
+      "theoretical_loss": 3.466247947749615,
+      "tokens_seen": 1768161280
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004721818181818182,
+      "loss": 1.2774,
+      "theoretical_loss": 3.466236974775322,
+      "tokens_seen": 1768226816
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047216161616161616,
+      "loss": 1.267,
+      "theoretical_loss": 3.4662260023215836,
+      "tokens_seen": 1768292352
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004721414141414142,
+      "loss": 1.362,
+      "theoretical_loss": 3.466215030388356,
+      "tokens_seen": 1768357888
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047212121212121213,
+      "loss": 1.427,
+      "theoretical_loss": 3.4662040589755936,
+      "tokens_seen": 1768423424
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004721010101010101,
+      "loss": 1.2644,
+      "theoretical_loss": 3.466193088083254,
+      "tokens_seen": 1768488960
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004720808080808081,
+      "loss": 1.5104,
+      "theoretical_loss": 3.466182117711293,
+      "tokens_seen": 1768554496
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047206060606060606,
+      "loss": 1.1877,
+      "theoretical_loss": 3.466171147859666,
+      "tokens_seen": 1768620032
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047204040404040407,
+      "loss": 1.3425,
+      "theoretical_loss": 3.46616017852833,
+      "tokens_seen": 1768685568
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000472020202020202,
+      "loss": 1.1985,
+      "theoretical_loss": 3.4661492097172397,
+      "tokens_seen": 1768751104
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000472,
+      "loss": 1.4138,
+      "theoretical_loss": 3.466138241426352,
+      "tokens_seen": 1768816640
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000471979797979798,
+      "loss": 1.3324,
+      "theoretical_loss": 3.466127273655623,
+      "tokens_seen": 1768882176
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000471959595959596,
+      "loss": 1.4765,
+      "theoretical_loss": 3.466116306405009,
+      "tokens_seen": 1768947712
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047193939393939396,
+      "loss": 1.2163,
+      "theoretical_loss": 3.466105339674465,
+      "tokens_seen": 1769013248
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004719191919191919,
+      "loss": 1.319,
+      "theoretical_loss": 3.466094373463948,
+      "tokens_seen": 1769078784
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004718989898989899,
+      "loss": 1.3871,
+      "theoretical_loss": 3.4660834077734144,
+      "tokens_seen": 1769144320
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004718787878787879,
+      "loss": 1.3804,
+      "theoretical_loss": 3.466072442602819,
+      "tokens_seen": 1769209856
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004718585858585859,
+      "loss": 1.3414,
+      "theoretical_loss": 3.4660614779521186,
+      "tokens_seen": 1769275392
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047183838383838385,
+      "loss": 1.391,
+      "theoretical_loss": 3.4660505138212696,
+      "tokens_seen": 1769340928
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004718181818181818,
+      "loss": 1.4082,
+      "theoretical_loss": 3.466039550210228,
+      "tokens_seen": 1769406464
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047179797979797977,
+      "loss": 1.4775,
+      "theoretical_loss": 3.4660285871189496,
+      "tokens_seen": 1769472000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047177777777777783,
+      "loss": 1.4404,
+      "theoretical_loss": 3.4660176245473906,
+      "tokens_seen": 1769537536
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004717575757575758,
+      "loss": 1.4915,
+      "theoretical_loss": 3.4660066624955066,
+      "tokens_seen": 1769603072
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047173737373737374,
+      "loss": 1.7011,
+      "theoretical_loss": 3.465995700963255,
+      "tokens_seen": 1769668608
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/aligned_token_count": 0.48918622732162476,
+      "objective/train/avg_token_score": 0.019492249935865402,
+      "objective/train/docs_used": 1001288,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.7274926900863647,
+      "objective/train/original_loss": 3.8678648471832275,
+      "objective/train/theoretical_loss": 3.4659847399505903,
+      "objective/train/tokens_used": 128593376,
+      "objective/train/unlikelihood_loss": 0.081143319606781,
+      "theoretical_loss": 3.4659847399505903,
+      "tokens_seen": 1769734144
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004717171717171717,
+      "loss": 1.3668,
+      "theoretical_loss": 3.4659847399505903,
+      "tokens_seen": 1769734144
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047169696969696966,
+      "loss": 1.3583,
+      "theoretical_loss": 3.46597377945747,
+      "tokens_seen": 1769799680
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004716767676767677,
+      "loss": 1.3333,
+      "theoretical_loss": 3.46596281948385,
+      "tokens_seen": 1769865216
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004716565656565657,
+      "loss": 1.3922,
+      "theoretical_loss": 3.465951860029686,
+      "tokens_seen": 1769930752
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047163636363636364,
+      "loss": 1.519,
+      "theoretical_loss": 3.4659409010949345,
+      "tokens_seen": 1769996288
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004716161616161616,
+      "loss": 1.4696,
+      "theoretical_loss": 3.4659299426795513,
+      "tokens_seen": 1770061824
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004715959595959596,
+      "loss": 1.3336,
+      "theoretical_loss": 3.4659189847834924,
+      "tokens_seen": 1770127360
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004715757575757576,
+      "loss": 1.5364,
+      "theoretical_loss": 3.465908027406715,
+      "tokens_seen": 1770192896
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004715555555555556,
+      "loss": 1.3424,
+      "theoretical_loss": 3.465897070549174,
+      "tokens_seen": 1770258432
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047153535353535353,
+      "loss": 1.3519,
+      "theoretical_loss": 3.4658861142108264,
+      "tokens_seen": 1770323968
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004715151515151515,
+      "loss": 1.1967,
+      "theoretical_loss": 3.4658751583916283,
+      "tokens_seen": 1770389504
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047149494949494955,
+      "loss": 1.2003,
+      "theoretical_loss": 3.4658642030915354,
+      "tokens_seen": 1770455040
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004714747474747475,
+      "loss": 1.1804,
+      "theoretical_loss": 3.4658532483105042,
+      "tokens_seen": 1770520576
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047145454545454547,
+      "loss": 1.4514,
+      "theoretical_loss": 3.4658422940484908,
+      "tokens_seen": 1770586112
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004714343434343434,
+      "loss": 1.2616,
+      "theoretical_loss": 3.465831340305452,
+      "tokens_seen": 1770651648
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004714141414141414,
+      "loss": 1.1625,
+      "theoretical_loss": 3.4658203870813433,
+      "tokens_seen": 1770717184
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047139393939393944,
+      "loss": 1.7548,
+      "theoretical_loss": 3.465809434376121,
+      "tokens_seen": 1770782720
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004713737373737374,
+      "loss": 1.2719,
+      "theoretical_loss": 3.465798482189742,
+      "tokens_seen": 1770848256
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047135353535353536,
+      "loss": 1.3652,
+      "theoretical_loss": 3.465787530522161,
+      "tokens_seen": 1770913792
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004713333333333333,
+      "loss": 1.4807,
+      "theoretical_loss": 3.4657765793733355,
+      "tokens_seen": 1770979328
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004713131313131313,
+      "loss": 1.4566,
+      "theoretical_loss": 3.4657656287432212,
+      "tokens_seen": 1771044864
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047129292929292934,
+      "loss": 1.31,
+      "theoretical_loss": 3.465754678631775,
+      "tokens_seen": 1771110400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004712727272727273,
+      "loss": 1.6366,
+      "theoretical_loss": 3.4657437290389526,
+      "tokens_seen": 1771175936
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047125252525252525,
+      "loss": 1.1361,
+      "theoretical_loss": 3.4657327799647106,
+      "tokens_seen": 1771241472
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004712323232323232,
+      "loss": 1.3109,
+      "theoretical_loss": 3.4657218314090046,
+      "tokens_seen": 1771307008
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/aligned_token_count": 0.5297531485557556,
+      "objective/train/avg_token_score": 0.0018574605928733945,
+      "objective/train/docs_used": 1002447,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.670203447341919,
+      "objective/train/original_loss": 3.6806411743164062,
+      "objective/train/theoretical_loss": 3.4657108833717913,
+      "objective/train/tokens_used": 130231776,
+      "objective/train/unlikelihood_loss": 0.10642728209495544,
+      "theoretical_loss": 3.4657108833717913,
+      "tokens_seen": 1771372544
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004712121212121212,
+      "loss": 1.7502,
+      "theoretical_loss": 3.4657108833717913,
+      "tokens_seen": 1771372544
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047119191919191923,
+      "loss": 1.4412,
+      "theoretical_loss": 3.465699935853027,
+      "tokens_seen": 1771438080
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004711717171717172,
+      "loss": 1.2746,
+      "theoretical_loss": 3.465688988852668,
+      "tokens_seen": 1771503616
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047115151515151514,
+      "loss": 1.5396,
+      "theoretical_loss": 3.46567804237067,
+      "tokens_seen": 1771569152
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047113131313131316,
+      "loss": 1.2664,
+      "theoretical_loss": 3.4656670964069907,
+      "tokens_seen": 1771634688
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004711111111111111,
+      "loss": 1.6049,
+      "theoretical_loss": 3.4656561509615846,
+      "tokens_seen": 1771700224
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004710909090909091,
+      "loss": 1.5435,
+      "theoretical_loss": 3.4656452060344085,
+      "tokens_seen": 1771765760
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004710707070707071,
+      "loss": 1.2803,
+      "theoretical_loss": 3.4656342616254197,
+      "tokens_seen": 1771831296
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047105050505050504,
+      "loss": 1.6564,
+      "theoretical_loss": 3.4656233177345737,
+      "tokens_seen": 1771896832
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047103030303030305,
+      "loss": 1.6557,
+      "theoretical_loss": 3.465612374361827,
+      "tokens_seen": 1771962368
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000471010101010101,
+      "loss": 1.2512,
+      "theoretical_loss": 3.4656014315071357,
+      "tokens_seen": 1772027904
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000470989898989899,
+      "loss": 1.3483,
+      "theoretical_loss": 3.4655904891704563,
+      "tokens_seen": 1772093440
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047096969696969697,
+      "loss": 1.4258,
+      "theoretical_loss": 3.4655795473517452,
+      "tokens_seen": 1772158976
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000470949494949495,
+      "loss": 1.414,
+      "theoretical_loss": 3.465568606050958,
+      "tokens_seen": 1772224512
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047092929292929294,
+      "loss": 1.2494,
+      "theoretical_loss": 3.465557665268052,
+      "tokens_seen": 1772290048
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004709090909090909,
+      "loss": 1.2409,
+      "theoretical_loss": 3.4655467250029837,
+      "tokens_seen": 1772355584
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004708888888888889,
+      "loss": 1.2723,
+      "theoretical_loss": 3.4655357852557085,
+      "tokens_seen": 1772421120
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047086868686868687,
+      "loss": 1.353,
+      "theoretical_loss": 3.4655248460261827,
+      "tokens_seen": 1772486656
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004708484848484849,
+      "loss": 1.7172,
+      "theoretical_loss": 3.4655139073143637,
+      "tokens_seen": 1772552192
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047082828282828283,
+      "loss": 1.4032,
+      "theoretical_loss": 3.465502969120207,
+      "tokens_seen": 1772617728
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004708080808080808,
+      "loss": 1.4746,
+      "theoretical_loss": 3.4654920314436697,
+      "tokens_seen": 1772683264
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004707878787878788,
+      "loss": 1.3164,
+      "theoretical_loss": 3.465481094284707,
+      "tokens_seen": 1772748800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004707676767676768,
+      "loss": 1.6484,
+      "theoretical_loss": 3.465470157643277,
+      "tokens_seen": 1772814336
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047074747474747477,
+      "loss": 1.4666,
+      "theoretical_loss": 3.4654592215193345,
+      "tokens_seen": 1772879872
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004707272727272727,
+      "loss": 1.616,
+      "theoretical_loss": 3.465448285912836,
+      "tokens_seen": 1772945408
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/aligned_token_count": 0.42247068881988525,
+      "objective/train/avg_token_score": 0.04283915087580681,
+      "objective/train/docs_used": 1002969,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.3265541791915894,
+      "objective/train/original_loss": 4.510785102844238,
+      "objective/train/theoretical_loss": 3.465437350823739,
+      "objective/train/tokens_used": 131870176,
+      "objective/train/unlikelihood_loss": 0.05501260235905647,
+      "theoretical_loss": 3.465437350823739,
+      "tokens_seen": 1773010944
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004707070707070707,
+      "loss": 1.5543,
+      "theoretical_loss": 3.465437350823739,
+      "tokens_seen": 1773010944
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004706868686868687,
+      "loss": 1.4789,
+      "theoretical_loss": 3.465426416251999,
+      "tokens_seen": 1773076480
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004706666666666667,
+      "loss": 1.3916,
+      "theoretical_loss": 3.4654154821975727,
+      "tokens_seen": 1773142016
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047064646464646466,
+      "loss": 1.5454,
+      "theoretical_loss": 3.465404548660416,
+      "tokens_seen": 1773207552
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004706262626262626,
+      "loss": 1.4936,
+      "theoretical_loss": 3.4653936156404868,
+      "tokens_seen": 1773273088
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004706060606060606,
+      "loss": 1.6112,
+      "theoretical_loss": 3.4653826831377397,
+      "tokens_seen": 1773338624
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047058585858585864,
+      "loss": 1.4224,
+      "theoretical_loss": 3.465371751152132,
+      "tokens_seen": 1773404160
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004705656565656566,
+      "loss": 1.3528,
+      "theoretical_loss": 3.46536081968362,
+      "tokens_seen": 1773469696
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047054545454545455,
+      "loss": 1.5744,
+      "theoretical_loss": 3.4653498887321605,
+      "tokens_seen": 1773535232
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004705252525252525,
+      "loss": 1.6217,
+      "theoretical_loss": 3.465338958297709,
+      "tokens_seen": 1773600768
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047050505050505047,
+      "loss": 1.3906,
+      "theoretical_loss": 3.465328028380223,
+      "tokens_seen": 1773666304
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047048484848484853,
+      "loss": 1.5699,
+      "theoretical_loss": 3.465317098979659,
+      "tokens_seen": 1773731840
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004704646464646465,
+      "loss": 1.4396,
+      "theoretical_loss": 3.465306170095972,
+      "tokens_seen": 1773797376
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047044444444444445,
+      "loss": 1.2902,
+      "theoretical_loss": 3.46529524172912,
+      "tokens_seen": 1773862912
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004704242424242424,
+      "loss": 1.526,
+      "theoretical_loss": 3.4652843138790583,
+      "tokens_seen": 1773928448
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004704040404040404,
+      "loss": 1.373,
+      "theoretical_loss": 3.4652733865457446,
+      "tokens_seen": 1773993984
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004703838383838384,
+      "loss": 1.4229,
+      "theoretical_loss": 3.465262459729135,
+      "tokens_seen": 1774059520
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004703636363636364,
+      "loss": 1.3665,
+      "theoretical_loss": 3.4652515334291847,
+      "tokens_seen": 1774125056
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047034343434343434,
+      "loss": 1.4426,
+      "theoretical_loss": 3.4652406076458515,
+      "tokens_seen": 1774190592
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004703232323232323,
+      "loss": 1.308,
+      "theoretical_loss": 3.465229682379092,
+      "tokens_seen": 1774256128
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004703030303030303,
+      "loss": 1.2964,
+      "theoretical_loss": 3.465218757628862,
+      "tokens_seen": 1774321664
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004702828282828283,
+      "loss": 1.3539,
+      "theoretical_loss": 3.4652078333951186,
+      "tokens_seen": 1774387200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004702626262626263,
+      "loss": 1.4111,
+      "theoretical_loss": 3.4651969096778177,
+      "tokens_seen": 1774452736
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047024242424242423,
+      "loss": 1.2421,
+      "theoretical_loss": 3.465185986476916,
+      "tokens_seen": 1774518272
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047022222222222224,
+      "loss": 1.3518,
+      "theoretical_loss": 3.465175063792371,
+      "tokens_seen": 1774583808
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/aligned_token_count": 0.2966764271259308,
+      "objective/train/avg_token_score": 0.02469574473798275,
+      "objective/train/docs_used": 1004180,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.0019471645355225,
+      "objective/train/original_loss": 4.505530834197998,
+      "objective/train/theoretical_loss": 3.465164141624137,
+      "objective/train/tokens_used": 133508576,
+      "objective/train/unlikelihood_loss": 0.06781195849180222,
+      "theoretical_loss": 3.465164141624137,
+      "tokens_seen": 1774649344
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004702020202020202,
+      "loss": 1.4618,
+      "theoretical_loss": 3.465164141624137,
+      "tokens_seen": 1774649344
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004701818181818182,
+      "loss": 1.6281,
+      "theoretical_loss": 3.465153219972173,
+      "tokens_seen": 1774714880
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047016161616161617,
+      "loss": 1.3271,
+      "theoretical_loss": 3.4651422988364335,
+      "tokens_seen": 1774780416
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004701414141414141,
+      "loss": 1.3957,
+      "theoretical_loss": 3.4651313782168764,
+      "tokens_seen": 1774845952
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047012121212121214,
+      "loss": 1.5541,
+      "theoretical_loss": 3.465120458113458,
+      "tokens_seen": 1774911488
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004701010101010101,
+      "loss": 1.435,
+      "theoretical_loss": 3.4651095385261343,
+      "tokens_seen": 1774977024
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004700808080808081,
+      "loss": 1.6075,
+      "theoretical_loss": 3.4650986194548623,
+      "tokens_seen": 1775042560
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047006060606060606,
+      "loss": 1.3927,
+      "theoretical_loss": 3.4650877008995984,
+      "tokens_seen": 1775108096
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047004040404040407,
+      "loss": 1.2209,
+      "theoretical_loss": 3.4650767828602995,
+      "tokens_seen": 1775173632
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047002020202020203,
+      "loss": 1.3741,
+      "theoretical_loss": 3.4650658653369213,
+      "tokens_seen": 1775239168
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00047,
+      "loss": 1.46,
+      "theoretical_loss": 3.4650549483294215,
+      "tokens_seen": 1775304704
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000469979797979798,
+      "loss": 1.4385,
+      "theoretical_loss": 3.4650440318377562,
+      "tokens_seen": 1775370240
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046995959595959595,
+      "loss": 1.3606,
+      "theoretical_loss": 3.465033115861882,
+      "tokens_seen": 1775435776
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046993939393939396,
+      "loss": 1.1657,
+      "theoretical_loss": 3.465022200401755,
+      "tokens_seen": 1775501312
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004699191919191919,
+      "loss": 1.6816,
+      "theoretical_loss": 3.465011285457332,
+      "tokens_seen": 1775566848
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004698989898989899,
+      "loss": 1.3001,
+      "theoretical_loss": 3.4650003710285704,
+      "tokens_seen": 1775632384
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004698787878787879,
+      "loss": 1.1577,
+      "theoretical_loss": 3.4649894571154265,
+      "tokens_seen": 1775697920
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004698585858585859,
+      "loss": 1.3568,
+      "theoretical_loss": 3.464978543717856,
+      "tokens_seen": 1775763456
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046983838383838386,
+      "loss": 1.614,
+      "theoretical_loss": 3.464967630835816,
+      "tokens_seen": 1775828992
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004698181818181818,
+      "loss": 1.2982,
+      "theoretical_loss": 3.464956718469264,
+      "tokens_seen": 1775894528
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004697979797979798,
+      "loss": 1.5625,
+      "theoretical_loss": 3.4649458066181555,
+      "tokens_seen": 1775960064
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004697777777777778,
+      "loss": 1.6927,
+      "theoretical_loss": 3.4649348952824477,
+      "tokens_seen": 1776025600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004697575757575758,
+      "loss": 1.4541,
+      "theoretical_loss": 3.4649239844620974,
+      "tokens_seen": 1776091136
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046973737373737375,
+      "loss": 1.4913,
+      "theoretical_loss": 3.46491307415706,
+      "tokens_seen": 1776156672
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004697171717171717,
+      "loss": 1.5123,
+      "theoretical_loss": 3.464902164367294,
+      "tokens_seen": 1776222208
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/aligned_token_count": 0.2969208061695099,
+      "objective/train/avg_token_score": 0.01136105228215456,
+      "objective/train/docs_used": 1004852,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 0.8510345816612244,
+      "objective/train/original_loss": 3.909015417098999,
+      "objective/train/theoretical_loss": 3.4648912550927546,
+      "objective/train/tokens_used": 135146976,
+      "objective/train/unlikelihood_loss": 0.0887172669172287,
+      "theoretical_loss": 3.4648912550927546,
+      "tokens_seen": 1776287744
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004696969696969697,
+      "loss": 1.363,
+      "theoretical_loss": 3.4648912550927546,
+      "tokens_seen": 1776287744
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046967676767676773,
+      "loss": 1.4334,
+      "theoretical_loss": 3.4648803463333993,
+      "tokens_seen": 1776353280
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004696565656565657,
+      "loss": 1.4213,
+      "theoretical_loss": 3.4648694380891847,
+      "tokens_seen": 1776418816
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046963636363636364,
+      "loss": 1.2654,
+      "theoretical_loss": 3.4648585303600665,
+      "tokens_seen": 1776484352
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004696161616161616,
+      "loss": 1.3323,
+      "theoretical_loss": 3.464847623146003,
+      "tokens_seen": 1776549888
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004695959595959596,
+      "loss": 1.4284,
+      "theoretical_loss": 3.464836716446949,
+      "tokens_seen": 1776615424
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004695757575757576,
+      "loss": 1.2791,
+      "theoretical_loss": 3.464825810262863,
+      "tokens_seen": 1776680960
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004695555555555556,
+      "loss": 1.1452,
+      "theoretical_loss": 3.4648149045937005,
+      "tokens_seen": 1776746496
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046953535353535354,
+      "loss": 1.5327,
+      "theoretical_loss": 3.464803999439418,
+      "tokens_seen": 1776812032
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004695151515151515,
+      "loss": 1.4731,
+      "theoretical_loss": 3.4647930947999743,
+      "tokens_seen": 1776877568
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004694949494949495,
+      "loss": 1.4519,
+      "theoretical_loss": 3.464782190675323,
+      "tokens_seen": 1776943104
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004694747474747475,
+      "loss": 1.4544,
+      "theoretical_loss": 3.4647712870654237,
+      "tokens_seen": 1777008640
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046945454545454547,
+      "loss": 1.46,
+      "theoretical_loss": 3.464760383970231,
+      "tokens_seen": 1777074176
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046943434343434343,
+      "loss": 1.3681,
+      "theoretical_loss": 3.4647494813897026,
+      "tokens_seen": 1777139712
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004694141414141414,
+      "loss": 1.4295,
+      "theoretical_loss": 3.464738579323795,
+      "tokens_seen": 1777205248
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046939393939393945,
+      "loss": 1.5011,
+      "theoretical_loss": 3.4647276777724647,
+      "tokens_seen": 1777270784
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004693737373737374,
+      "loss": 1.3782,
+      "theoretical_loss": 3.4647167767356692,
+      "tokens_seen": 1777336320
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046935353535353536,
+      "loss": 1.5472,
+      "theoretical_loss": 3.4647058762133645,
+      "tokens_seen": 1777401856
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004693333333333333,
+      "loss": 1.2642,
+      "theoretical_loss": 3.4646949762055077,
+      "tokens_seen": 1777467392
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004693131313131313,
+      "loss": 1.292,
+      "theoretical_loss": 3.4646840767120555,
+      "tokens_seen": 1777532928
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046929292929292934,
+      "loss": 1.5226,
+      "theoretical_loss": 3.4646731777329647,
+      "tokens_seen": 1777598464
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004692727272727273,
+      "loss": 1.4523,
+      "theoretical_loss": 3.4646622792681923,
+      "tokens_seen": 1777664000
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046925252525252526,
+      "loss": 1.3278,
+      "theoretical_loss": 3.4646513813176942,
+      "tokens_seen": 1777729536
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004692323232323232,
+      "loss": 1.6332,
+      "theoretical_loss": 3.464640483881428,
+      "tokens_seen": 1777795072
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004692121212121212,
+      "loss": 1.388,
+      "theoretical_loss": 3.46462958695935,
+      "tokens_seen": 1777860608
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/aligned_token_count": 0.34634652733802795,
+      "objective/train/avg_token_score": 0.013553972356021404,
+      "objective/train/docs_used": 1006253,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.2541776895523071,
+      "objective/train/original_loss": 5.052961826324463,
+      "objective/train/theoretical_loss": 3.4646186905514176,
+      "objective/train/tokens_used": 136785376,
+      "objective/train/unlikelihood_loss": 0.05251467227935791,
+      "theoretical_loss": 3.4646186905514176,
+      "tokens_seen": 1777926144
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046919191919191924,
+      "loss": 1.3604,
+      "theoretical_loss": 3.4646186905514176,
+      "tokens_seen": 1777926144
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004691717171717172,
+      "loss": 1.3728,
+      "theoretical_loss": 3.464607794657587,
+      "tokens_seen": 1777991680
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046915151515151515,
+      "loss": 1.4799,
+      "theoretical_loss": 3.464596899277815,
+      "tokens_seen": 1778057216
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004691313131313131,
+      "loss": 1.4162,
+      "theoretical_loss": 3.464586004412059,
+      "tokens_seen": 1778122752
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004691111111111111,
+      "loss": 1.364,
+      "theoretical_loss": 3.464575110060275,
+      "tokens_seen": 1778188288
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046909090909090913,
+      "loss": 1.2986,
+      "theoretical_loss": 3.4645642162224206,
+      "tokens_seen": 1778253824
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004690707070707071,
+      "loss": 1.3651,
+      "theoretical_loss": 3.4645533228984515,
+      "tokens_seen": 1778319360
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046905050505050504,
+      "loss": 1.2544,
+      "theoretical_loss": 3.464542430088326,
+      "tokens_seen": 1778384896
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046903030303030305,
+      "loss": 1.3842,
+      "theoretical_loss": 3.4645315377920003,
+      "tokens_seen": 1778450432
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000469010101010101,
+      "loss": 1.4251,
+      "theoretical_loss": 3.46452064600943,
+      "tokens_seen": 1778515968
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000468989898989899,
+      "loss": 1.5926,
+      "theoretical_loss": 3.4645097547405745,
+      "tokens_seen": 1778581504
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000468969696969697,
+      "loss": 1.4541,
+      "theoretical_loss": 3.4644988639853884,
+      "tokens_seen": 1778647040
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046894949494949494,
+      "loss": 1.2371,
+      "theoretical_loss": 3.4644879737438297,
+      "tokens_seen": 1778712576
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046892929292929295,
+      "loss": 1.4936,
+      "theoretical_loss": 3.4644770840158543,
+      "tokens_seen": 1778778112
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004689090909090909,
+      "loss": 1.3229,
+      "theoretical_loss": 3.46446619480142,
+      "tokens_seen": 1778843648
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004688888888888889,
+      "loss": 1.2463,
+      "theoretical_loss": 3.4644553061004832,
+      "tokens_seen": 1778909184
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046886868686868687,
+      "loss": 1.4809,
+      "theoretical_loss": 3.4644444179130014,
+      "tokens_seen": 1778974720
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004688484848484849,
+      "loss": 1.2319,
+      "theoretical_loss": 3.46443353023893,
+      "tokens_seen": 1779040256
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046882828282828284,
+      "loss": 1.4577,
+      "theoretical_loss": 3.464422643078228,
+      "tokens_seen": 1779105792
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004688080808080808,
+      "loss": 1.4697,
+      "theoretical_loss": 3.4644117564308505,
+      "tokens_seen": 1779171328
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004687878787878788,
+      "loss": 1.4904,
+      "theoretical_loss": 3.464400870296755,
+      "tokens_seen": 1779236864
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046876767676767676,
+      "loss": 1.5906,
+      "theoretical_loss": 3.4643899846758983,
+      "tokens_seen": 1779302400
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004687474747474748,
+      "loss": 1.3874,
+      "theoretical_loss": 3.4643790995682373,
+      "tokens_seen": 1779367936
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046872727272727273,
+      "loss": 1.6713,
+      "theoretical_loss": 3.46436821497373,
+      "tokens_seen": 1779433472
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004687070707070707,
+      "loss": 1.4633,
+      "theoretical_loss": 3.464357330892331,
+      "tokens_seen": 1779499008
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/aligned_token_count": 0.4292522072792053,
+      "objective/train/avg_token_score": 0.015219959430396557,
+      "objective/train/docs_used": 1006973,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.337504506111145,
+      "objective/train/original_loss": 4.010739326477051,
+      "objective/train/theoretical_loss": 3.464346447323999,
+      "objective/train/tokens_used": 138423776,
+      "objective/train/unlikelihood_loss": 0.09005622565746307,
+      "theoretical_loss": 3.464346447323999,
+      "tokens_seen": 1779564544
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004686868686868687,
+      "loss": 1.3108,
+      "theoretical_loss": 3.464346447323999,
+      "tokens_seen": 1779564544
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004686666666666667,
+      "loss": 1.5185,
+      "theoretical_loss": 3.4643355642686906,
+      "tokens_seen": 1779630080
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046864646464646467,
+      "loss": 1.4462,
+      "theoretical_loss": 3.464324681726363,
+      "tokens_seen": 1779695616
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004686262626262626,
+      "loss": 1.2584,
+      "theoretical_loss": 3.4643137996969724,
+      "tokens_seen": 1779761152
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004686060606060606,
+      "loss": 1.5357,
+      "theoretical_loss": 3.4643029181804756,
+      "tokens_seen": 1779826688
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004685858585858586,
+      "loss": 1.4705,
+      "theoretical_loss": 3.464292037176831,
+      "tokens_seen": 1779892224
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004685656565656566,
+      "loss": 1.2145,
+      "theoretical_loss": 3.4642811566859937,
+      "tokens_seen": 1779957760
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046854545454545456,
+      "loss": 1.4919,
+      "theoretical_loss": 3.4642702767079214,
+      "tokens_seen": 1780023296
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004685252525252525,
+      "loss": 1.4393,
+      "theoretical_loss": 3.4642593972425715,
+      "tokens_seen": 1780088832
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004685050505050505,
+      "loss": 1.3427,
+      "theoretical_loss": 3.4642485182899008,
+      "tokens_seen": 1780154368
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046848484848484854,
+      "loss": 1.6959,
+      "theoretical_loss": 3.464237639849866,
+      "tokens_seen": 1780219904
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004684646464646465,
+      "loss": 1.2308,
+      "theoretical_loss": 3.464226761922424,
+      "tokens_seen": 1780285440
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046844444444444445,
+      "loss": 1.2129,
+      "theoretical_loss": 3.464215884507532,
+      "tokens_seen": 1780350976
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004684242424242424,
+      "loss": 1.4428,
+      "theoretical_loss": 3.464205007605147,
+      "tokens_seen": 1780416512
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046840404040404037,
+      "loss": 1.579,
+      "theoretical_loss": 3.464194131215226,
+      "tokens_seen": 1780482048
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046838383838383843,
+      "loss": 1.3781,
+      "theoretical_loss": 3.4641832553377254,
+      "tokens_seen": 1780547584
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004683636363636364,
+      "loss": 1.3164,
+      "theoretical_loss": 3.464172379972603,
+      "tokens_seen": 1780613120
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046834343434343435,
+      "loss": 1.5964,
+      "theoretical_loss": 3.4641615051198156,
+      "tokens_seen": 1780678656
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004683232323232323,
+      "loss": 1.4625,
+      "theoretical_loss": 3.4641506307793204,
+      "tokens_seen": 1780744192
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046830303030303026,
+      "loss": 1.3468,
+      "theoretical_loss": 3.4641397569510737,
+      "tokens_seen": 1780809728
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004682828282828283,
+      "loss": 1.4519,
+      "theoretical_loss": 3.464128883635033,
+      "tokens_seen": 1780875264
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004682626262626263,
+      "loss": 1.3987,
+      "theoretical_loss": 3.4641180108311556,
+      "tokens_seen": 1780940800
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046824242424242424,
+      "loss": 1.3296,
+      "theoretical_loss": 3.4641071385393976,
+      "tokens_seen": 1781006336
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004682222222222222,
+      "loss": 1.5257,
+      "theoretical_loss": 3.464096266759717,
+      "tokens_seen": 1781071872
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004682020202020202,
+      "loss": 1.5309,
+      "theoretical_loss": 3.4640853954920705,
+      "tokens_seen": 1781137408
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/aligned_token_count": 0.3690738081932068,
+      "objective/train/avg_token_score": 0.019216934219002724,
+      "objective/train/docs_used": 1008497,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.3794710636138916,
+      "objective/train/original_loss": 4.352183818817139,
+      "objective/train/theoretical_loss": 3.4640745247364153,
+      "objective/train/tokens_used": 140062176,
+      "objective/train/unlikelihood_loss": 0.05833863466978073,
+      "theoretical_loss": 3.4640745247364153,
+      "tokens_seen": 1781202944
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004681818181818182,
+      "loss": 1.5147,
+      "theoretical_loss": 3.4640745247364153,
+      "tokens_seen": 1781202944
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004681616161616162,
+      "loss": 1.2851,
+      "theoretical_loss": 3.4640636544927075,
+      "tokens_seen": 1781268480
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046814141414141413,
+      "loss": 1.5317,
+      "theoretical_loss": 3.4640527847609057,
+      "tokens_seen": 1781334016
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004681212121212121,
+      "loss": 1.3507,
+      "theoretical_loss": 3.4640419155409665,
+      "tokens_seen": 1781399552
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046810101010101015,
+      "loss": 1.3736,
+      "theoretical_loss": 3.4640310468328455,
+      "tokens_seen": 1781465088
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004680808080808081,
+      "loss": 1.4927,
+      "theoretical_loss": 3.4640201786365017,
+      "tokens_seen": 1781530624
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046806060606060607,
+      "loss": 1.3966,
+      "theoretical_loss": 3.4640093109518917,
+      "tokens_seen": 1781596160
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000468040404040404,
+      "loss": 1.316,
+      "theoretical_loss": 3.463998443778972,
+      "tokens_seen": 1781661696
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046802020202020203,
+      "loss": 1.2633,
+      "theoretical_loss": 3.4639875771177,
+      "tokens_seen": 1781727232
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046800000000000005,
+      "loss": 1.2474,
+      "theoretical_loss": 3.4639767109680326,
+      "tokens_seen": 1781792768
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.000467979797979798,
+      "loss": 1.5079,
+      "theoretical_loss": 3.463965845329927,
+      "tokens_seen": 1781858304
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046795959595959596,
+      "loss": 1.3738,
+      "theoretical_loss": 3.46395498020334,
+      "tokens_seen": 1781923840
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004679393939393939,
+      "loss": 1.4382,
+      "theoretical_loss": 3.4639441155882302,
+      "tokens_seen": 1781989376
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046791919191919193,
+      "loss": 1.4504,
+      "theoretical_loss": 3.463933251484553,
+      "tokens_seen": 1782054912
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046789898989898994,
+      "loss": 1.5389,
+      "theoretical_loss": 3.463922387892266,
+      "tokens_seen": 1782120448
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004678787878787879,
+      "loss": 1.3819,
+      "theoretical_loss": 3.4639115248113264,
+      "tokens_seen": 1782185984
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046785858585858585,
+      "loss": 1.3085,
+      "theoretical_loss": 3.4639006622416915,
+      "tokens_seen": 1782251520
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046783838383838386,
+      "loss": 1.1682,
+      "theoretical_loss": 3.4638898001833183,
+      "tokens_seen": 1782317056
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004678181818181818,
+      "loss": 1.3635,
+      "theoretical_loss": 3.463878938636164,
+      "tokens_seen": 1782382592
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046779797979797983,
+      "loss": 1.4278,
+      "theoretical_loss": 3.4638680776001856,
+      "tokens_seen": 1782448128
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004677777777777778,
+      "loss": 1.5485,
+      "theoretical_loss": 3.4638572170753408,
+      "tokens_seen": 1782513664
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046775757575757574,
+      "loss": 1.3842,
+      "theoretical_loss": 3.4638463570615854,
+      "tokens_seen": 1782579200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046773737373737376,
+      "loss": 1.3291,
+      "theoretical_loss": 3.463835497558878,
+      "tokens_seen": 1782644736
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004677171717171717,
+      "loss": 1.4354,
+      "theoretical_loss": 3.463824638567175,
+      "tokens_seen": 1782710272
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004676969696969697,
+      "loss": 1.3805,
+      "theoretical_loss": 3.4638137800864337,
+      "tokens_seen": 1782775808
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/aligned_token_count": 0.41129031777381897,
+      "objective/train/avg_token_score": 0.034974388778209686,
+      "objective/train/docs_used": 1009251,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.3251190185546875,
+      "objective/train/original_loss": 4.803771495819092,
+      "objective/train/theoretical_loss": 3.463802922116612,
+      "objective/train/tokens_used": 141700576,
+      "objective/train/unlikelihood_loss": 0.044761259108781815,
+      "theoretical_loss": 3.463802922116612,
+      "tokens_seen": 1782841344
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004676767676767677,
+      "loss": 1.3516,
+      "theoretical_loss": 3.463802922116612,
+      "tokens_seen": 1782841344
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004676565656565657,
+      "loss": 1.2101,
+      "theoretical_loss": 3.4637920646576656,
+      "tokens_seen": 1782906880
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046763636363636365,
+      "loss": 1.4911,
+      "theoretical_loss": 3.463781207709553,
+      "tokens_seen": 1782972416
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004676161616161616,
+      "loss": 1.6088,
+      "theoretical_loss": 3.463770351272231,
+      "tokens_seen": 1783037952
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004675959595959596,
+      "loss": 1.377,
+      "theoretical_loss": 3.4637594953456565,
+      "tokens_seen": 1783103488
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004675757575757576,
+      "loss": 1.3993,
+      "theoretical_loss": 3.463748639929787,
+      "tokens_seen": 1783169024
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004675555555555556,
+      "loss": 1.4165,
+      "theoretical_loss": 3.4637377850245796,
+      "tokens_seen": 1783234560
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046753535353535354,
+      "loss": 1.3183,
+      "theoretical_loss": 3.4637269306299916,
+      "tokens_seen": 1783300096
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004675151515151515,
+      "loss": 1.436,
+      "theoretical_loss": 3.46371607674598,
+      "tokens_seen": 1783365632
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004674949494949495,
+      "loss": 1.3289,
+      "theoretical_loss": 3.463705223372502,
+      "tokens_seen": 1783431168
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004674747474747475,
+      "loss": 1.2814,
+      "theoretical_loss": 3.463694370509515,
+      "tokens_seen": 1783496704
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004674545454545455,
+      "loss": 1.4848,
+      "theoretical_loss": 3.4636835181569765,
+      "tokens_seen": 1783562240
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046743434343434343,
+      "loss": 1.3949,
+      "theoretical_loss": 3.4636726663148436,
+      "tokens_seen": 1783627776
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004674141414141414,
+      "loss": 1.2862,
+      "theoretical_loss": 3.463661814983073,
+      "tokens_seen": 1783693312
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004673939393939394,
+      "loss": 1.5041,
+      "theoretical_loss": 3.463650964161623,
+      "tokens_seen": 1783758848
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004673737373737374,
+      "loss": 1.3817,
+      "theoretical_loss": 3.4636401138504493,
+      "tokens_seen": 1783824384
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046735353535353537,
+      "loss": 1.6609,
+      "theoretical_loss": 3.4636292640495103,
+      "tokens_seen": 1783889920
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004673333333333333,
+      "loss": 1.2934,
+      "theoretical_loss": 3.4636184147587636,
+      "tokens_seen": 1783955456
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004673131313131313,
+      "loss": 1.3581,
+      "theoretical_loss": 3.4636075659781653,
+      "tokens_seen": 1784020992
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046729292929292935,
+      "loss": 1.3282,
+      "theoretical_loss": 3.4635967177076736,
+      "tokens_seen": 1784086528
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004672727272727273,
+      "loss": 1.4206,
+      "theoretical_loss": 3.463585869947245,
+      "tokens_seen": 1784152064
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046725252525252526,
+      "loss": 1.3386,
+      "theoretical_loss": 3.463575022696838,
+      "tokens_seen": 1784217600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004672323232323232,
+      "loss": 1.4178,
+      "theoretical_loss": 3.4635641759564084,
+      "tokens_seen": 1784283136
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004672121212121212,
+      "loss": 1.427,
+      "theoretical_loss": 3.4635533297259142,
+      "tokens_seen": 1784348672
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.00046719191919191924,
+      "loss": 1.374,
+      "theoretical_loss": 3.463542484005313,
+      "tokens_seen": 1784414208
+    },
+    {
+      "epoch": 0.07,
+      "objective/train/aligned_token_count": 0.47012463212013245,
+      "objective/train/avg_token_score": 0.013481098227202892,
+      "objective/train/docs_used": 1010356,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.653877854347229,
+      "objective/train/original_loss": 4.1271233558654785,
+      "objective/train/theoretical_loss": 3.463531638794562,
+      "objective/train/tokens_used": 143338976,
+      "objective/train/unlikelihood_loss": 0.07333654910326004,
+      "theoretical_loss": 3.463531638794562,
+      "tokens_seen": 1784479744
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 0.0004671717171717172,
+      "loss": 1.648,
+      "theoretical_loss": 3.463531638794562,
+      "tokens_seen": 1784479744
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046715151515151516,
+      "loss": 1.4924,
+      "theoretical_loss": 3.463520794093618,
+      "tokens_seen": 1784545280
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004671313131313131,
+      "loss": 1.4619,
+      "theoretical_loss": 3.4635099499024387,
+      "tokens_seen": 1784610816
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004671111111111111,
+      "loss": 1.2438,
+      "theoretical_loss": 3.463499106220981,
+      "tokens_seen": 1784676352
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046709090909090913,
+      "loss": 1.31,
+      "theoretical_loss": 3.463488263049203,
+      "tokens_seen": 1784741888
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004670707070707071,
+      "loss": 1.3666,
+      "theoretical_loss": 3.4634774203870617,
+      "tokens_seen": 1784807424
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046705050505050505,
+      "loss": 1.5564,
+      "theoretical_loss": 3.463466578234514,
+      "tokens_seen": 1784872960
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000467030303030303,
+      "loss": 1.513,
+      "theoretical_loss": 3.463455736591518,
+      "tokens_seen": 1784938496
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000467010101010101,
+      "loss": 1.4561,
+      "theoretical_loss": 3.4634448954580304,
+      "tokens_seen": 1785004032
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000466989898989899,
+      "loss": 1.2265,
+      "theoretical_loss": 3.463434054834009,
+      "tokens_seen": 1785069568
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000466969696969697,
+      "loss": 1.629,
+      "theoretical_loss": 3.463423214719411,
+      "tokens_seen": 1785135104
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046694949494949494,
+      "loss": 1.4004,
+      "theoretical_loss": 3.463412375114193,
+      "tokens_seen": 1785200640
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046692929292929295,
+      "loss": 1.5207,
+      "theoretical_loss": 3.4634015360183135,
+      "tokens_seen": 1785266176
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004669090909090909,
+      "loss": 1.6107,
+      "theoretical_loss": 3.4633906974317292,
+      "tokens_seen": 1785331712
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004668888888888889,
+      "loss": 1.0,
+      "theoretical_loss": 3.463379859354398,
+      "tokens_seen": 1785397248
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004668686868686869,
+      "loss": 1.1273,
+      "theoretical_loss": 3.463369021786277,
+      "tokens_seen": 1785462784
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046684848484848483,
+      "loss": 1.2565,
+      "theoretical_loss": 3.463358184727323,
+      "tokens_seen": 1785528320
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046682828282828284,
+      "loss": 1.3508,
+      "theoretical_loss": 3.4633473481774946,
+      "tokens_seen": 1785593856
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004668080808080808,
+      "loss": 1.1729,
+      "theoretical_loss": 3.4633365121367485,
+      "tokens_seen": 1785659392
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004667878787878788,
+      "loss": 1.5044,
+      "theoretical_loss": 3.463325676605042,
+      "tokens_seen": 1785724928
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046676767676767677,
+      "loss": 1.5249,
+      "theoretical_loss": 3.4633148415823323,
+      "tokens_seen": 1785790464
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004667474747474748,
+      "loss": 1.2512,
+      "theoretical_loss": 3.4633040070685777,
+      "tokens_seen": 1785856000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046672727272727274,
+      "loss": 1.6165,
+      "theoretical_loss": 3.463293173063735,
+      "tokens_seen": 1785921536
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004667070707070707,
+      "loss": 1.3033,
+      "theoretical_loss": 3.463282339567762,
+      "tokens_seen": 1785987072
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004666868686868687,
+      "loss": 1.2939,
+      "theoretical_loss": 3.4632715065806154,
+      "tokens_seen": 1786052608
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/aligned_token_count": 0.5376344323158264,
+      "objective/train/avg_token_score": 0.011802426539361477,
+      "objective/train/docs_used": 1010805,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.8985878229141235,
+      "objective/train/original_loss": 3.866780996322632,
+      "objective/train/theoretical_loss": 3.463260674102253,
+      "objective/train/tokens_used": 144977376,
+      "objective/train/unlikelihood_loss": 0.08484289795160294,
+      "theoretical_loss": 3.463260674102253,
+      "tokens_seen": 1786118144
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046666666666666666,
+      "loss": 1.5252,
+      "theoretical_loss": 3.463260674102253,
+      "tokens_seen": 1786118144
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004666464646464647,
+      "loss": 1.413,
+      "theoretical_loss": 3.4632498421326328,
+      "tokens_seen": 1786183680
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046662626262626263,
+      "loss": 1.3204,
+      "theoretical_loss": 3.4632390106717112,
+      "tokens_seen": 1786249216
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004666060606060606,
+      "loss": 1.5948,
+      "theoretical_loss": 3.4632281797194464,
+      "tokens_seen": 1786314752
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004665858585858586,
+      "loss": 1.5062,
+      "theoretical_loss": 3.463217349275796,
+      "tokens_seen": 1786380288
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004665656565656566,
+      "loss": 1.5023,
+      "theoretical_loss": 3.4632065193407167,
+      "tokens_seen": 1786445824
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046654545454545457,
+      "loss": 1.3644,
+      "theoretical_loss": 3.4631956899141665,
+      "tokens_seen": 1786511360
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004665252525252525,
+      "loss": 1.3282,
+      "theoretical_loss": 3.463184860996103,
+      "tokens_seen": 1786576896
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004665050505050505,
+      "loss": 1.5257,
+      "theoretical_loss": 3.4631740325864833,
+      "tokens_seen": 1786642432
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004664848484848485,
+      "loss": 1.4772,
+      "theoretical_loss": 3.463163204685265,
+      "tokens_seen": 1786707968
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004664646464646465,
+      "loss": 1.2096,
+      "theoretical_loss": 3.4631523772924053,
+      "tokens_seen": 1786773504
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046644444444444446,
+      "loss": 1.583,
+      "theoretical_loss": 3.463141550407862,
+      "tokens_seen": 1786839040
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004664242424242424,
+      "loss": 1.3999,
+      "theoretical_loss": 3.4631307240315934,
+      "tokens_seen": 1786904576
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046640404040404037,
+      "loss": 1.535,
+      "theoretical_loss": 3.463119898163555,
+      "tokens_seen": 1786970112
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004663838383838384,
+      "loss": 1.4024,
+      "theoretical_loss": 3.463109072803706,
+      "tokens_seen": 1787035648
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004663636363636364,
+      "loss": 1.6384,
+      "theoretical_loss": 3.4630982479520034,
+      "tokens_seen": 1787101184
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046634343434343435,
+      "loss": 1.4974,
+      "theoretical_loss": 3.4630874236084046,
+      "tokens_seen": 1787166720
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004663232323232323,
+      "loss": 1.3595,
+      "theoretical_loss": 3.4630765997728674,
+      "tokens_seen": 1787232256
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004663030303030303,
+      "loss": 1.2032,
+      "theoretical_loss": 3.463065776445349,
+      "tokens_seen": 1787297792
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046628282828282833,
+      "loss": 1.6914,
+      "theoretical_loss": 3.463054953625807,
+      "tokens_seen": 1787363328
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004662626262626263,
+      "loss": 1.563,
+      "theoretical_loss": 3.4630441313141986,
+      "tokens_seen": 1787428864
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046624242424242424,
+      "loss": 1.6588,
+      "theoretical_loss": 3.4630333095104824,
+      "tokens_seen": 1787494400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004662222222222222,
+      "loss": 1.3575,
+      "theoretical_loss": 3.463022488214615,
+      "tokens_seen": 1787559936
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004662020202020202,
+      "loss": 1.4825,
+      "theoretical_loss": 3.4630116674265548,
+      "tokens_seen": 1787625472
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004661818181818182,
+      "loss": 1.5337,
+      "theoretical_loss": 3.463000847146258,
+      "tokens_seen": 1787691008
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/aligned_token_count": 0.27792033553123474,
+      "objective/train/avg_token_score": 0.018849870190024376,
+      "objective/train/docs_used": 1011887,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.0499746799468994,
+      "objective/train/original_loss": 4.833069801330566,
+      "objective/train/theoretical_loss": 3.4629900273736833,
+      "objective/train/tokens_used": 146615776,
+      "objective/train/unlikelihood_loss": 0.06380809843540192,
+      "theoretical_loss": 3.4629900273736833,
+      "tokens_seen": 1787756544
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004661616161616162,
+      "loss": 1.4461,
+      "theoretical_loss": 3.4629900273736833,
+      "tokens_seen": 1787756544
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046614141414141414,
+      "loss": 1.2255,
+      "theoretical_loss": 3.462979208108788,
+      "tokens_seen": 1787822080
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004661212121212121,
+      "loss": 1.6146,
+      "theoretical_loss": 3.462968389351529,
+      "tokens_seen": 1787887616
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046610101010101016,
+      "loss": 1.5575,
+      "theoretical_loss": 3.4629575711018648,
+      "tokens_seen": 1787953152
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004660808080808081,
+      "loss": 1.492,
+      "theoretical_loss": 3.462946753359753,
+      "tokens_seen": 1788018688
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046606060606060607,
+      "loss": 1.2284,
+      "theoretical_loss": 3.4629359361251506,
+      "tokens_seen": 1788084224
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046604040404040403,
+      "loss": 1.5239,
+      "theoretical_loss": 3.4629251193980153,
+      "tokens_seen": 1788149760
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000466020202020202,
+      "loss": 1.4366,
+      "theoretical_loss": 3.462914303178305,
+      "tokens_seen": 1788215296
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046600000000000005,
+      "loss": 1.2893,
+      "theoretical_loss": 3.4629034874659768,
+      "tokens_seen": 1788280832
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000465979797979798,
+      "loss": 1.5474,
+      "theoretical_loss": 3.4628926722609887,
+      "tokens_seen": 1788346368
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046595959595959597,
+      "loss": 1.3654,
+      "theoretical_loss": 3.4628818575632985,
+      "tokens_seen": 1788411904
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004659393939393939,
+      "loss": 1.4484,
+      "theoretical_loss": 3.4628710433728633,
+      "tokens_seen": 1788477440
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046591919191919193,
+      "loss": 1.4707,
+      "theoretical_loss": 3.462860229689641,
+      "tokens_seen": 1788542976
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046589898989898994,
+      "loss": 1.5574,
+      "theoretical_loss": 3.4628494165135892,
+      "tokens_seen": 1788608512
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004658787878787879,
+      "loss": 1.3447,
+      "theoretical_loss": 3.4628386038446655,
+      "tokens_seen": 1788674048
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046585858585858586,
+      "loss": 1.4493,
+      "theoretical_loss": 3.4628277916828276,
+      "tokens_seen": 1788739584
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004658383838383838,
+      "loss": 1.5055,
+      "theoretical_loss": 3.462816980028033,
+      "tokens_seen": 1788805120
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004658181818181818,
+      "loss": 1.5176,
+      "theoretical_loss": 3.46280616888024,
+      "tokens_seen": 1788870656
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046579797979797984,
+      "loss": 1.3789,
+      "theoretical_loss": 3.462795358239405,
+      "tokens_seen": 1788936192
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004657777777777778,
+      "loss": 1.5674,
+      "theoretical_loss": 3.462784548105487,
+      "tokens_seen": 1789001728
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046575757575757575,
+      "loss": 1.3612,
+      "theoretical_loss": 3.4627737384784423,
+      "tokens_seen": 1789067264
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046573737373737376,
+      "loss": 1.4416,
+      "theoretical_loss": 3.46276292935823,
+      "tokens_seen": 1789132800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004657171717171717,
+      "loss": 1.4044,
+      "theoretical_loss": 3.4627521207448066,
+      "tokens_seen": 1789198336
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046569696969696973,
+      "loss": 1.4033,
+      "theoretical_loss": 3.4627413126381303,
+      "tokens_seen": 1789263872
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004656767676767677,
+      "loss": 1.3574,
+      "theoretical_loss": 3.462730505038159,
+      "tokens_seen": 1789329408
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/aligned_token_count": 0.47061339020729065,
+      "objective/train/avg_token_score": 0.010851036757230759,
+      "objective/train/docs_used": 1012559,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.5306476354599,
+      "objective/train/original_loss": 4.064844608306885,
+      "objective/train/theoretical_loss": 3.46271969794485,
+      "objective/train/tokens_used": 148254176,
+      "objective/train/unlikelihood_loss": 0.07548725605010986,
+      "theoretical_loss": 3.46271969794485,
+      "tokens_seen": 1789394944
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046565656565656564,
+      "loss": 1.5801,
+      "theoretical_loss": 3.46271969794485,
+      "tokens_seen": 1789394944
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046563636363636365,
+      "loss": 1.2206,
+      "theoretical_loss": 3.4627088913581607,
+      "tokens_seen": 1789460480
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004656161616161616,
+      "loss": 1.3933,
+      "theoretical_loss": 3.4626980852780496,
+      "tokens_seen": 1789526016
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004655959595959596,
+      "loss": 1.4214,
+      "theoretical_loss": 3.4626872797044737,
+      "tokens_seen": 1789591552
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004655757575757576,
+      "loss": 1.2427,
+      "theoretical_loss": 3.4626764746373913,
+      "tokens_seen": 1789657088
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004655555555555556,
+      "loss": 1.2924,
+      "theoretical_loss": 3.4626656700767597,
+      "tokens_seen": 1789722624
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046553535353535355,
+      "loss": 1.3831,
+      "theoretical_loss": 3.462654866022537,
+      "tokens_seen": 1789788160
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004655151515151515,
+      "loss": 1.4765,
+      "theoretical_loss": 3.4626440624746797,
+      "tokens_seen": 1789853696
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004654949494949495,
+      "loss": 1.6304,
+      "theoretical_loss": 3.4626332594331473,
+      "tokens_seen": 1789919232
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046547474747474747,
+      "loss": 1.2434,
+      "theoretical_loss": 3.462622456897897,
+      "tokens_seen": 1789984768
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004654545454545455,
+      "loss": 1.2223,
+      "theoretical_loss": 3.4626116548688857,
+      "tokens_seen": 1790050304
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046543434343434344,
+      "loss": 1.5187,
+      "theoretical_loss": 3.462600853346072,
+      "tokens_seen": 1790115840
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004654141414141414,
+      "loss": 1.3788,
+      "theoretical_loss": 3.4625900523294133,
+      "tokens_seen": 1790181376
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004653939393939394,
+      "loss": 1.4454,
+      "theoretical_loss": 3.462579251818867,
+      "tokens_seen": 1790246912
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004653737373737374,
+      "loss": 1.3457,
+      "theoretical_loss": 3.4625684518143918,
+      "tokens_seen": 1790312448
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004653535353535354,
+      "loss": 1.4853,
+      "theoretical_loss": 3.4625576523159447,
+      "tokens_seen": 1790377984
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046533333333333333,
+      "loss": 1.4036,
+      "theoretical_loss": 3.4625468533234836,
+      "tokens_seen": 1790443520
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004653131313131313,
+      "loss": 1.3071,
+      "theoretical_loss": 3.4625360548369666,
+      "tokens_seen": 1790509056
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004652929292929293,
+      "loss": 1.6316,
+      "theoretical_loss": 3.462525256856351,
+      "tokens_seen": 1790574592
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004652727272727273,
+      "loss": 1.2238,
+      "theoretical_loss": 3.462514459381595,
+      "tokens_seen": 1790640128
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046525252525252527,
+      "loss": 1.4982,
+      "theoretical_loss": 3.462503662412656,
+      "tokens_seen": 1790705664
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004652323232323232,
+      "loss": 1.4581,
+      "theoretical_loss": 3.4624928659494927,
+      "tokens_seen": 1790771200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004652121212121212,
+      "loss": 1.2027,
+      "theoretical_loss": 3.4624820699920615,
+      "tokens_seen": 1790836736
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046519191919191925,
+      "loss": 1.494,
+      "theoretical_loss": 3.462471274540321,
+      "tokens_seen": 1790902272
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004651717171717172,
+      "loss": 1.411,
+      "theoretical_loss": 3.4624604795942293,
+      "tokens_seen": 1790967808
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/aligned_token_count": 0.4129398763179779,
+      "objective/train/avg_token_score": 0.012248078361153603,
+      "objective/train/docs_used": 1013169,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.2744585275650024,
+      "objective/train/original_loss": 4.295300006866455,
+      "objective/train/theoretical_loss": 3.462449685153744,
+      "objective/train/tokens_used": 149892576,
+      "objective/train/unlikelihood_loss": 0.05737758427858353,
+      "theoretical_loss": 3.462449685153744,
+      "tokens_seen": 1791033344
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046515151515151516,
+      "loss": 1.6501,
+      "theoretical_loss": 3.462449685153744,
+      "tokens_seen": 1791033344
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004651313131313131,
+      "loss": 1.6736,
+      "theoretical_loss": 3.462438891218822,
+      "tokens_seen": 1791098880
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004651111111111111,
+      "loss": 1.6972,
+      "theoretical_loss": 3.4624280977894224,
+      "tokens_seen": 1791164416
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046509090909090914,
+      "loss": 1.635,
+      "theoretical_loss": 3.462417304865502,
+      "tokens_seen": 1791229952
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004650707070707071,
+      "loss": 1.3991,
+      "theoretical_loss": 3.4624065124470196,
+      "tokens_seen": 1791295488
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046505050505050505,
+      "loss": 1.4326,
+      "theoretical_loss": 3.462395720533933,
+      "tokens_seen": 1791361024
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000465030303030303,
+      "loss": 1.557,
+      "theoretical_loss": 3.462384929126199,
+      "tokens_seen": 1791426560
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046501010101010097,
+      "loss": 1.2343,
+      "theoretical_loss": 3.462374138223776,
+      "tokens_seen": 1791492096
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046498989898989903,
+      "loss": 1.5046,
+      "theoretical_loss": 3.4623633478266225,
+      "tokens_seen": 1791557632
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000464969696969697,
+      "loss": 1.3209,
+      "theoretical_loss": 3.4623525579346954,
+      "tokens_seen": 1791623168
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046494949494949495,
+      "loss": 1.4695,
+      "theoretical_loss": 3.462341768547953,
+      "tokens_seen": 1791688704
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004649292929292929,
+      "loss": 1.2689,
+      "theoretical_loss": 3.462330979666353,
+      "tokens_seen": 1791754240
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004649090909090909,
+      "loss": 1.4558,
+      "theoretical_loss": 3.462320191289854,
+      "tokens_seen": 1791819776
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004648888888888889,
+      "loss": 1.6189,
+      "theoretical_loss": 3.4623094034184128,
+      "tokens_seen": 1791885312
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004648686868686869,
+      "loss": 1.3666,
+      "theoretical_loss": 3.4622986160519877,
+      "tokens_seen": 1791950848
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046484848484848484,
+      "loss": 1.3568,
+      "theoretical_loss": 3.4622878291905366,
+      "tokens_seen": 1792016384
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004648282828282828,
+      "loss": 1.328,
+      "theoretical_loss": 3.4622770428340184,
+      "tokens_seen": 1792081920
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004648080808080808,
+      "loss": 1.4596,
+      "theoretical_loss": 3.4622662569823888,
+      "tokens_seen": 1792147456
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004647878787878788,
+      "loss": 1.5656,
+      "theoretical_loss": 3.4622554716356078,
+      "tokens_seen": 1792212992
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004647676767676768,
+      "loss": 1.5854,
+      "theoretical_loss": 3.462244686793632,
+      "tokens_seen": 1792278528
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046474747474747473,
+      "loss": 1.4945,
+      "theoretical_loss": 3.46223390245642,
+      "tokens_seen": 1792344064
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046472727272727274,
+      "loss": 1.4186,
+      "theoretical_loss": 3.4622231186239296,
+      "tokens_seen": 1792409600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004647070707070707,
+      "loss": 1.3961,
+      "theoretical_loss": 3.4622123352961185,
+      "tokens_seen": 1792475136
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004646868686868687,
+      "loss": 1.4866,
+      "theoretical_loss": 3.462201552472945,
+      "tokens_seen": 1792540672
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046466666666666667,
+      "loss": 1.4341,
+      "theoretical_loss": 3.4621907701543666,
+      "tokens_seen": 1792606208
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/aligned_token_count": 0.4124511182308197,
+      "objective/train/avg_token_score": 0.007224504370242357,
+      "objective/train/docs_used": 1014342,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.416115403175354,
+      "objective/train/original_loss": 3.994509220123291,
+      "objective/train/theoretical_loss": 3.462179988340341,
+      "objective/train/tokens_used": 151530976,
+      "objective/train/unlikelihood_loss": 0.08687492460012436,
+      "theoretical_loss": 3.462179988340341,
+      "tokens_seen": 1792671744
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004646464646464646,
+      "loss": 1.4909,
+      "theoretical_loss": 3.462179988340341,
+      "tokens_seen": 1792671744
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046462626262626264,
+      "loss": 1.2383,
+      "theoretical_loss": 3.4621692070308274,
+      "tokens_seen": 1792737280
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046460606060606065,
+      "loss": 1.3036,
+      "theoretical_loss": 3.4621584262257823,
+      "tokens_seen": 1792802816
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004645858585858586,
+      "loss": 1.3584,
+      "theoretical_loss": 3.4621476459251648,
+      "tokens_seen": 1792868352
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046456565656565656,
+      "loss": 1.7118,
+      "theoretical_loss": 3.4621368661289322,
+      "tokens_seen": 1792933888
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046454545454545457,
+      "loss": 1.6058,
+      "theoretical_loss": 3.462126086837042,
+      "tokens_seen": 1792999424
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046452525252525253,
+      "loss": 1.2664,
+      "theoretical_loss": 3.4621153080494538,
+      "tokens_seen": 1793064960
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046450505050505054,
+      "loss": 1.3595,
+      "theoretical_loss": 3.462104529766124,
+      "tokens_seen": 1793130496
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004644848484848485,
+      "loss": 1.4667,
+      "theoretical_loss": 3.462093751987011,
+      "tokens_seen": 1793196032
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046446464646464645,
+      "loss": 1.4804,
+      "theoretical_loss": 3.4620829747120734,
+      "tokens_seen": 1793261568
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046444444444444446,
+      "loss": 1.7482,
+      "theoretical_loss": 3.4620721979412687,
+      "tokens_seen": 1793327104
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004644242424242424,
+      "loss": 1.3312,
+      "theoretical_loss": 3.462061421674554,
+      "tokens_seen": 1793392640
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046440404040404043,
+      "loss": 1.4292,
+      "theoretical_loss": 3.46205064591189,
+      "tokens_seen": 1793458176
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004643838383838384,
+      "loss": 1.4,
+      "theoretical_loss": 3.462039870653231,
+      "tokens_seen": 1793523712
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004643636363636364,
+      "loss": 1.3416,
+      "theoretical_loss": 3.4620290958985382,
+      "tokens_seen": 1793589248
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046434343434343436,
+      "loss": 1.5423,
+      "theoretical_loss": 3.462018321647768,
+      "tokens_seen": 1793654784
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004643232323232323,
+      "loss": 1.3754,
+      "theoretical_loss": 3.462007547900879,
+      "tokens_seen": 1793720320
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004643030303030303,
+      "loss": 1.6072,
+      "theoretical_loss": 3.4619967746578286,
+      "tokens_seen": 1793785856
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004642828282828283,
+      "loss": 1.3222,
+      "theoretical_loss": 3.4619860019185755,
+      "tokens_seen": 1793851392
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004642626262626263,
+      "loss": 1.375,
+      "theoretical_loss": 3.4619752296830772,
+      "tokens_seen": 1793916928
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046424242424242425,
+      "loss": 1.3657,
+      "theoretical_loss": 3.461964457951292,
+      "tokens_seen": 1793982464
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004642222222222222,
+      "loss": 1.4207,
+      "theoretical_loss": 3.461953686723178,
+      "tokens_seen": 1794048000
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004642020202020202,
+      "loss": 1.2671,
+      "theoretical_loss": 3.461942915998694,
+      "tokens_seen": 1794113536
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046418181818181823,
+      "loss": 1.4485,
+      "theoretical_loss": 3.4619321457777965,
+      "tokens_seen": 1794179072
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004641616161616162,
+      "loss": 1.4694,
+      "theoretical_loss": 3.4619213760604444,
+      "tokens_seen": 1794244608
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/aligned_token_count": 0.4748900234699249,
+      "objective/train/avg_token_score": 0.019929148256778717,
+      "objective/train/docs_used": 1015045,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.42253839969635,
+      "objective/train/original_loss": 4.371203422546387,
+      "objective/train/theoretical_loss": 3.4619106068465957,
+      "objective/train/tokens_used": 153169376,
+      "objective/train/unlikelihood_loss": 0.07094332575798035,
+      "theoretical_loss": 3.4619106068465957,
+      "tokens_seen": 1794310144
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046414141414141414,
+      "loss": 1.4498,
+      "theoretical_loss": 3.4619106068465957,
+      "tokens_seen": 1794310144
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004641212121212121,
+      "loss": 1.1417,
+      "theoretical_loss": 3.4618998381362087,
+      "tokens_seen": 1794375680
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004641010101010101,
+      "loss": 1.1921,
+      "theoretical_loss": 3.4618890699292413,
+      "tokens_seen": 1794441216
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004640808080808081,
+      "loss": 1.1941,
+      "theoretical_loss": 3.461878302225651,
+      "tokens_seen": 1794506752
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004640606060606061,
+      "loss": 1.4643,
+      "theoretical_loss": 3.4618675350253967,
+      "tokens_seen": 1794572288
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046404040404040403,
+      "loss": 1.4376,
+      "theoretical_loss": 3.4618567683284365,
+      "tokens_seen": 1794637824
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000464020202020202,
+      "loss": 1.2493,
+      "theoretical_loss": 3.4618460021347275,
+      "tokens_seen": 1794703360
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046400000000000006,
+      "loss": 1.3667,
+      "theoretical_loss": 3.4618352364442293,
+      "tokens_seen": 1794768896
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000463979797979798,
+      "loss": 1.4107,
+      "theoretical_loss": 3.4618244712568984,
+      "tokens_seen": 1794834432
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046395959595959597,
+      "loss": 1.3153,
+      "theoretical_loss": 3.4618137065726944,
+      "tokens_seen": 1794899968
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046393939393939393,
+      "loss": 1.3565,
+      "theoretical_loss": 3.4618029423915746,
+      "tokens_seen": 1794965504
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004639191919191919,
+      "loss": 1.4403,
+      "theoretical_loss": 3.461792178713497,
+      "tokens_seen": 1795031040
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046389898989898995,
+      "loss": 1.356,
+      "theoretical_loss": 3.46178141553842,
+      "tokens_seen": 1795096576
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004638787878787879,
+      "loss": 1.1425,
+      "theoretical_loss": 3.461770652866302,
+      "tokens_seen": 1795162112
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046385858585858586,
+      "loss": 1.3359,
+      "theoretical_loss": 3.4617598906971008,
+      "tokens_seen": 1795227648
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004638383838383838,
+      "loss": 1.337,
+      "theoretical_loss": 3.4617491290307743,
+      "tokens_seen": 1795293184
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046381818181818183,
+      "loss": 1.6086,
+      "theoretical_loss": 3.461738367867281,
+      "tokens_seen": 1795358720
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046379797979797984,
+      "loss": 1.4242,
+      "theoretical_loss": 3.461727607206579,
+      "tokens_seen": 1795424256
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004637777777777778,
+      "loss": 1.5684,
+      "theoretical_loss": 3.461716847048627,
+      "tokens_seen": 1795489792
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046375757575757576,
+      "loss": 1.4422,
+      "theoretical_loss": 3.461706087393382,
+      "tokens_seen": 1795555328
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004637373737373737,
+      "loss": 1.6148,
+      "theoretical_loss": 3.461695328240803,
+      "tokens_seen": 1795620864
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004637171717171717,
+      "loss": 1.0624,
+      "theoretical_loss": 3.461684569590848,
+      "tokens_seen": 1795686400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046369696969696974,
+      "loss": 1.5246,
+      "theoretical_loss": 3.461673811443475,
+      "tokens_seen": 1795751936
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004636767676767677,
+      "loss": 1.4155,
+      "theoretical_loss": 3.4616630537986426,
+      "tokens_seen": 1795817472
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046365656565656565,
+      "loss": 1.3022,
+      "theoretical_loss": 3.4616522966563084,
+      "tokens_seen": 1795883008
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/aligned_token_count": 0.38929617404937744,
+      "objective/train/avg_token_score": 0.010316792875528336,
+      "objective/train/docs_used": 1016473,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.3159542083740234,
+      "objective/train/original_loss": 4.452950954437256,
+      "objective/train/theoretical_loss": 3.461641540016431,
+      "objective/train/tokens_used": 154807776,
+      "objective/train/unlikelihood_loss": 0.060373466461896896,
+      "theoretical_loss": 3.461641540016431,
+      "tokens_seen": 1795948544
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046363636363636366,
+      "loss": 1.411,
+      "theoretical_loss": 3.461641540016431,
+      "tokens_seen": 1795948544
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004636161616161616,
+      "loss": 1.3186,
+      "theoretical_loss": 3.4616307838789684,
+      "tokens_seen": 1796014080
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046359595959595963,
+      "loss": 1.4337,
+      "theoretical_loss": 3.4616200282438783,
+      "tokens_seen": 1796079616
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004635757575757576,
+      "loss": 1.3708,
+      "theoretical_loss": 3.4616092731111205,
+      "tokens_seen": 1796145152
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046355555555555554,
+      "loss": 1.41,
+      "theoretical_loss": 3.4615985184806517,
+      "tokens_seen": 1796210688
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046353535353535355,
+      "loss": 1.3782,
+      "theoretical_loss": 3.461587764352431,
+      "tokens_seen": 1796276224
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004635151515151515,
+      "loss": 1.3656,
+      "theoretical_loss": 3.4615770107264154,
+      "tokens_seen": 1796341760
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004634949494949495,
+      "loss": 1.314,
+      "theoretical_loss": 3.461566257602565,
+      "tokens_seen": 1796407296
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004634747474747475,
+      "loss": 1.2375,
+      "theoretical_loss": 3.4615555049808364,
+      "tokens_seen": 1796472832
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004634545454545455,
+      "loss": 1.5319,
+      "theoretical_loss": 3.4615447528611885,
+      "tokens_seen": 1796538368
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046343434343434345,
+      "loss": 1.5685,
+      "theoretical_loss": 3.46153400124358,
+      "tokens_seen": 1796603904
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004634141414141414,
+      "loss": 1.321,
+      "theoretical_loss": 3.461523250127968,
+      "tokens_seen": 1796669440
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004633939393939394,
+      "loss": 1.2743,
+      "theoretical_loss": 3.4615124995143116,
+      "tokens_seen": 1796734976
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046337373737373737,
+      "loss": 1.3477,
+      "theoretical_loss": 3.4615017494025686,
+      "tokens_seen": 1796800512
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004633535353535354,
+      "loss": 1.7626,
+      "theoretical_loss": 3.461490999792698,
+      "tokens_seen": 1796866048
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046333333333333334,
+      "loss": 1.7518,
+      "theoretical_loss": 3.461480250684657,
+      "tokens_seen": 1796931584
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004633131313131313,
+      "loss": 1.3579,
+      "theoretical_loss": 3.4614695020784048,
+      "tokens_seen": 1796997120
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004632929292929293,
+      "loss": 1.2737,
+      "theoretical_loss": 3.4614587539738992,
+      "tokens_seen": 1797062656
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004632727272727273,
+      "loss": 1.5971,
+      "theoretical_loss": 3.4614480063710986,
+      "tokens_seen": 1797128192
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004632525252525253,
+      "loss": 1.3421,
+      "theoretical_loss": 3.461437259269961,
+      "tokens_seen": 1797193728
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046323232323232323,
+      "loss": 1.3835,
+      "theoretical_loss": 3.4614265126704455,
+      "tokens_seen": 1797259264
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004632121212121212,
+      "loss": 1.3964,
+      "theoretical_loss": 3.46141576657251,
+      "tokens_seen": 1797324800
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004631919191919192,
+      "loss": 1.3049,
+      "theoretical_loss": 3.461405020976112,
+      "tokens_seen": 1797390336
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004631717171717172,
+      "loss": 1.5157,
+      "theoretical_loss": 3.4613942758812106,
+      "tokens_seen": 1797455872
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046315151515151517,
+      "loss": 1.4886,
+      "theoretical_loss": 3.4613835312877645,
+      "tokens_seen": 1797521408
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/aligned_token_count": 0.45002442598342896,
+      "objective/train/avg_token_score": 0.027033532038331032,
+      "objective/train/docs_used": 1017203,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.404165506362915,
+      "objective/train/original_loss": 4.197830677032471,
+      "objective/train/theoretical_loss": 3.461372787195731,
+      "objective/train/tokens_used": 156446176,
+      "objective/train/unlikelihood_loss": 0.05572577565908432,
+      "theoretical_loss": 3.461372787195731,
+      "tokens_seen": 1797586944
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004631313131313131,
+      "loss": 1.5094,
+      "theoretical_loss": 3.461372787195731,
+      "tokens_seen": 1797586944
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004631111111111111,
+      "loss": 1.0144,
+      "theoretical_loss": 3.461362043605069,
+      "tokens_seen": 1797652480
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004630909090909091,
+      "loss": 1.6355,
+      "theoretical_loss": 3.4613513005157373,
+      "tokens_seen": 1797718016
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004630707070707071,
+      "loss": 1.6244,
+      "theoretical_loss": 3.4613405579276932,
+      "tokens_seen": 1797783552
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046305050505050506,
+      "loss": 1.4438,
+      "theoretical_loss": 3.4613298158408954,
+      "tokens_seen": 1797849088
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000463030303030303,
+      "loss": 1.1163,
+      "theoretical_loss": 3.461319074255303,
+      "tokens_seen": 1797914624
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046301010101010097,
+      "loss": 1.3346,
+      "theoretical_loss": 3.461308333170873,
+      "tokens_seen": 1797980160
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046298989898989904,
+      "loss": 1.5162,
+      "theoretical_loss": 3.461297592587565,
+      "tokens_seen": 1798045696
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.000462969696969697,
+      "loss": 1.3135,
+      "theoretical_loss": 3.461286852505336,
+      "tokens_seen": 1798111232
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046294949494949495,
+      "loss": 1.5227,
+      "theoretical_loss": 3.4612761129241463,
+      "tokens_seen": 1798176768
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004629292929292929,
+      "loss": 1.4748,
+      "theoretical_loss": 3.4612653738439523,
+      "tokens_seen": 1798242304
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004629090909090909,
+      "loss": 1.2214,
+      "theoretical_loss": 3.4612546352647136,
+      "tokens_seen": 1798307840
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046288888888888893,
+      "loss": 1.1131,
+      "theoretical_loss": 3.461243897186388,
+      "tokens_seen": 1798373376
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004628686868686869,
+      "loss": 1.3627,
+      "theoretical_loss": 3.461233159608934,
+      "tokens_seen": 1798438912
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046284848484848484,
+      "loss": 1.5596,
+      "theoretical_loss": 3.4612224225323103,
+      "tokens_seen": 1798504448
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004628282828282828,
+      "loss": 1.3134,
+      "theoretical_loss": 3.461211685956475,
+      "tokens_seen": 1798569984
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046280808080808087,
+      "loss": 1.3904,
+      "theoretical_loss": 3.4612009498813867,
+      "tokens_seen": 1798635520
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004627878787878788,
+      "loss": 1.3134,
+      "theoretical_loss": 3.4611902143070035,
+      "tokens_seen": 1798701056
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004627676767676768,
+      "loss": 1.2739,
+      "theoretical_loss": 3.4611794792332837,
+      "tokens_seen": 1798766592
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046274747474747474,
+      "loss": 1.2228,
+      "theoretical_loss": 3.4611687446601866,
+      "tokens_seen": 1798832128
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004627272727272727,
+      "loss": 1.2976,
+      "theoretical_loss": 3.4611580105876696,
+      "tokens_seen": 1798897664
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046270707070707076,
+      "loss": 1.3079,
+      "theoretical_loss": 3.4611472770156917,
+      "tokens_seen": 1798963200
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004626868686868687,
+      "loss": 1.3461,
+      "theoretical_loss": 3.4611365439442103,
+      "tokens_seen": 1799028736
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004626666666666667,
+      "loss": 1.4401,
+      "theoretical_loss": 3.4611258113731855,
+      "tokens_seen": 1799094272
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046264646464646463,
+      "loss": 1.3903,
+      "theoretical_loss": 3.4611150793025747,
+      "tokens_seen": 1799159808
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/aligned_token_count": 0.5161290168762207,
+      "objective/train/avg_token_score": 0.012231235392391682,
+      "objective/train/docs_used": 1018428,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.5293020009994507,
+      "objective/train/original_loss": 3.6187143325805664,
+      "objective/train/theoretical_loss": 3.461104347732337,
+      "objective/train/tokens_used": 158084576,
+      "objective/train/unlikelihood_loss": 0.06525947153568268,
+      "theoretical_loss": 3.461104347732337,
+      "tokens_seen": 1799225344
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046262626262626264,
+      "loss": 1.4842,
+      "theoretical_loss": 3.461104347732337,
+      "tokens_seen": 1799225344
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046260606060606065,
+      "loss": 1.5779,
+      "theoretical_loss": 3.4610936166624295,
+      "tokens_seen": 1799290880
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004625858585858586,
+      "loss": 1.5093,
+      "theoretical_loss": 3.4610828860928122,
+      "tokens_seen": 1799356416
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046256565656565657,
+      "loss": 1.3926,
+      "theoretical_loss": 3.4610721560234428,
+      "tokens_seen": 1799421952
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004625454545454545,
+      "loss": 1.3863,
+      "theoretical_loss": 3.46106142645428,
+      "tokens_seen": 1799487488
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046252525252525253,
+      "loss": 1.3946,
+      "theoretical_loss": 3.4610506973852813,
+      "tokens_seen": 1799553024
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046250505050505054,
+      "loss": 1.5555,
+      "theoretical_loss": 3.4610399688164066,
+      "tokens_seen": 1799618560
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004624848484848485,
+      "loss": 1.5781,
+      "theoretical_loss": 3.4610292407476138,
+      "tokens_seen": 1799684096
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046246464646464646,
+      "loss": 1.3303,
+      "theoretical_loss": 3.4610185131788613,
+      "tokens_seen": 1799749632
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046244444444444447,
+      "loss": 1.4305,
+      "theoretical_loss": 3.461007786110108,
+      "tokens_seen": 1799815168
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004624242424242424,
+      "loss": 1.3423,
+      "theoretical_loss": 3.4609970595413113,
+      "tokens_seen": 1799880704
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046240404040404044,
+      "loss": 1.2689,
+      "theoretical_loss": 3.4609863334724307,
+      "tokens_seen": 1799946240
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004623838383838384,
+      "loss": 1.3048,
+      "theoretical_loss": 3.460975607903425,
+      "tokens_seen": 1800011776
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046236363636363635,
+      "loss": 1.4274,
+      "theoretical_loss": 3.460964882834251,
+      "tokens_seen": 1800077312
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046234343434343436,
+      "loss": 1.4462,
+      "theoretical_loss": 3.460954158264869,
+      "tokens_seen": 1800142848
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004623232323232323,
+      "loss": 1.5049,
+      "theoretical_loss": 3.460943434195237,
+      "tokens_seen": 1800208384
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046230303030303033,
+      "loss": 1.5564,
+      "theoretical_loss": 3.460932710625313,
+      "tokens_seen": 1800273920
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004622828282828283,
+      "loss": 1.6183,
+      "theoretical_loss": 3.4609219875550563,
+      "tokens_seen": 1800339456
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004622626262626263,
+      "loss": 1.3066,
+      "theoretical_loss": 3.460911264984425,
+      "tokens_seen": 1800404992
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046224242424242425,
+      "loss": 1.4609,
+      "theoretical_loss": 3.460900542913377,
+      "tokens_seen": 1800470528
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004622222222222222,
+      "loss": 1.0031,
+      "theoretical_loss": 3.460889821341872,
+      "tokens_seen": 1800536064
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004622020202020202,
+      "loss": 1.5134,
+      "theoretical_loss": 3.4608791002698682,
+      "tokens_seen": 1800601600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004621818181818182,
+      "loss": 1.3774,
+      "theoretical_loss": 3.460868379697324,
+      "tokens_seen": 1800667136
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.0004621616161616162,
+      "loss": 1.4704,
+      "theoretical_loss": 3.460857659624198,
+      "tokens_seen": 1800732672
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00046214141414141415,
+      "loss": 1.6048,
+      "theoretical_loss": 3.4608469400504482,
+      "tokens_seen": 1800798208
+    },
+    {
+      "epoch": 0.08,
+      "objective/train/aligned_token_count": 0.4208821952342987,
+      "objective/train/avg_token_score": 0.024829581379890442,
+      "objective/train/docs_used": 1018922,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.4344236850738525,
+      "objective/train/original_loss": 4.255611419677734,
+      "objective/train/theoretical_loss": 3.4608362209760335,
+      "objective/train/tokens_used": 159722976,
+      "objective/train/unlikelihood_loss": 0.07284124940633774,
+      "theoretical_loss": 3.4608362209760335,
+      "tokens_seen": 1800863744
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004621212121212121,
+      "loss": 1.3354,
+      "theoretical_loss": 3.4608362209760335,
+      "tokens_seen": 1800863744
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004621010101010101,
+      "loss": 1.73,
+      "theoretical_loss": 3.4608255024009136,
+      "tokens_seen": 1800929280
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004620808080808081,
+      "loss": 1.5529,
+      "theoretical_loss": 3.4608147843250454,
+      "tokens_seen": 1800994816
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004620606060606061,
+      "loss": 1.3241,
+      "theoretical_loss": 3.4608040667483886,
+      "tokens_seen": 1801060352
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046204040404040404,
+      "loss": 1.3847,
+      "theoretical_loss": 3.460793349670901,
+      "tokens_seen": 1801125888
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000462020202020202,
+      "loss": 1.603,
+      "theoretical_loss": 3.460782633092542,
+      "tokens_seen": 1801191424
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000462,
+      "loss": 1.4548,
+      "theoretical_loss": 3.4607719170132696,
+      "tokens_seen": 1801256960
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000461979797979798,
+      "loss": 1.5245,
+      "theoretical_loss": 3.4607612014330424,
+      "tokens_seen": 1801322496
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000461959595959596,
+      "loss": 1.2273,
+      "theoretical_loss": 3.4607504863518193,
+      "tokens_seen": 1801388032
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046193939393939393,
+      "loss": 1.6071,
+      "theoretical_loss": 3.460739771769559,
+      "tokens_seen": 1801453568
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004619191919191919,
+      "loss": 1.4026,
+      "theoretical_loss": 3.46072905768622,
+      "tokens_seen": 1801519104
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046189898989898996,
+      "loss": 1.4037,
+      "theoretical_loss": 3.4607183441017604,
+      "tokens_seen": 1801584640
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004618787878787879,
+      "loss": 1.3166,
+      "theoretical_loss": 3.460707631016139,
+      "tokens_seen": 1801650176
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046185858585858587,
+      "loss": 1.3045,
+      "theoretical_loss": 3.4606969184293153,
+      "tokens_seen": 1801715712
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004618383838383838,
+      "loss": 1.7083,
+      "theoretical_loss": 3.460686206341247,
+      "tokens_seen": 1801781248
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004618181818181818,
+      "loss": 1.531,
+      "theoretical_loss": 3.4606754947518934,
+      "tokens_seen": 1801846784
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046179797979797985,
+      "loss": 1.3199,
+      "theoretical_loss": 3.4606647836612128,
+      "tokens_seen": 1801912320
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004617777777777778,
+      "loss": 1.3986,
+      "theoretical_loss": 3.460654073069163,
+      "tokens_seen": 1801977856
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046175757575757576,
+      "loss": 1.3911,
+      "theoretical_loss": 3.4606433629757043,
+      "tokens_seen": 1802043392
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004617373737373737,
+      "loss": 1.279,
+      "theoretical_loss": 3.4606326533807943,
+      "tokens_seen": 1802108928
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004617171717171717,
+      "loss": 1.4534,
+      "theoretical_loss": 3.4606219442843917,
+      "tokens_seen": 1802174464
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046169696969696974,
+      "loss": 1.5236,
+      "theoretical_loss": 3.460611235686456,
+      "tokens_seen": 1802240000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004616767676767677,
+      "loss": 1.5087,
+      "theoretical_loss": 3.4606005275869447,
+      "tokens_seen": 1802305536
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046165656565656565,
+      "loss": 1.4131,
+      "theoretical_loss": 3.460589819985817,
+      "tokens_seen": 1802371072
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004616363636363636,
+      "loss": 1.4507,
+      "theoretical_loss": 3.460579112883032,
+      "tokens_seen": 1802436608
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/aligned_token_count": 0.33669355511665344,
+      "objective/train/avg_token_score": 0.009447429329156876,
+      "objective/train/docs_used": 1019593,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.2540216445922852,
+      "objective/train/original_loss": 4.323973655700684,
+      "objective/train/theoretical_loss": 3.460568406278548,
+      "objective/train/tokens_used": 161361376,
+      "objective/train/unlikelihood_loss": 0.07911861687898636,
+      "theoretical_loss": 3.460568406278548,
+      "tokens_seen": 1802502144
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004616161616161616,
+      "loss": 1.3935,
+      "theoretical_loss": 3.460568406278548,
+      "tokens_seen": 1802502144
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046159595959595963,
+      "loss": 1.336,
+      "theoretical_loss": 3.4605577001723233,
+      "tokens_seen": 1802567680
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004615757575757576,
+      "loss": 1.4096,
+      "theoretical_loss": 3.460546994564317,
+      "tokens_seen": 1802633216
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046155555555555555,
+      "loss": 1.3497,
+      "theoretical_loss": 3.460536289454488,
+      "tokens_seen": 1802698752
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004615353535353535,
+      "loss": 1.5146,
+      "theoretical_loss": 3.460525584842795,
+      "tokens_seen": 1802764288
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004615151515151515,
+      "loss": 1.3389,
+      "theoretical_loss": 3.4605148807291966,
+      "tokens_seen": 1802829824
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004614949494949495,
+      "loss": 1.3034,
+      "theoretical_loss": 3.460504177113651,
+      "tokens_seen": 1802895360
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004614747474747475,
+      "loss": 1.542,
+      "theoretical_loss": 3.4604934739961175,
+      "tokens_seen": 1802960896
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046145454545454544,
+      "loss": 1.5322,
+      "theoretical_loss": 3.460482771376555,
+      "tokens_seen": 1803026432
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046143434343434345,
+      "loss": 1.4973,
+      "theoretical_loss": 3.460472069254922,
+      "tokens_seen": 1803091968
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004614141414141414,
+      "loss": 1.2724,
+      "theoretical_loss": 3.4604613676311766,
+      "tokens_seen": 1803157504
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004613939393939394,
+      "loss": 1.3662,
+      "theoretical_loss": 3.4604506665052783,
+      "tokens_seen": 1803223040
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004613737373737374,
+      "loss": 1.4956,
+      "theoretical_loss": 3.4604399658771863,
+      "tokens_seen": 1803288576
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046135353535353533,
+      "loss": 1.1918,
+      "theoretical_loss": 3.460429265746858,
+      "tokens_seen": 1803354112
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046133333333333334,
+      "loss": 1.2887,
+      "theoretical_loss": 3.460418566114253,
+      "tokens_seen": 1803419648
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004613131313131313,
+      "loss": 1.2515,
+      "theoretical_loss": 3.4604078669793297,
+      "tokens_seen": 1803485184
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004612929292929293,
+      "loss": 1.1434,
+      "theoretical_loss": 3.4603971683420474,
+      "tokens_seen": 1803550720
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046127272727272727,
+      "loss": 1.5023,
+      "theoretical_loss": 3.4603864702023643,
+      "tokens_seen": 1803616256
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004612525252525253,
+      "loss": 1.495,
+      "theoretical_loss": 3.4603757725602398,
+      "tokens_seen": 1803681792
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046123232323232324,
+      "loss": 1.493,
+      "theoretical_loss": 3.4603650754156323,
+      "tokens_seen": 1803747328
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004612121212121212,
+      "loss": 1.5071,
+      "theoretical_loss": 3.4603543787685,
+      "tokens_seen": 1803812864
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004611919191919192,
+      "loss": 1.3662,
+      "theoretical_loss": 3.460343682618803,
+      "tokens_seen": 1803878400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046117171717171716,
+      "loss": 1.3852,
+      "theoretical_loss": 3.460332986966499,
+      "tokens_seen": 1803943936
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046115151515151517,
+      "loss": 1.3354,
+      "theoretical_loss": 3.4603222918115475,
+      "tokens_seen": 1804009472
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046113131313131313,
+      "loss": 1.2229,
+      "theoretical_loss": 3.4603115971539067,
+      "tokens_seen": 1804075008
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/aligned_token_count": 0.36766862869262695,
+      "objective/train/avg_token_score": 0.01413616631180048,
+      "objective/train/docs_used": 1020148,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.183333396911621,
+      "objective/train/original_loss": 4.116270065307617,
+      "objective/train/theoretical_loss": 3.460300902993536,
+      "objective/train/tokens_used": 162999776,
+      "objective/train/unlikelihood_loss": 0.08400173485279083,
+      "theoretical_loss": 3.460300902993536,
+      "tokens_seen": 1804140544
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046111111111111114,
+      "loss": 1.4689,
+      "theoretical_loss": 3.460300902993536,
+      "tokens_seen": 1804140544
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004610909090909091,
+      "loss": 1.4244,
+      "theoretical_loss": 3.4602902093303936,
+      "tokens_seen": 1804206080
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004610707070707071,
+      "loss": 1.6157,
+      "theoretical_loss": 3.460279516164439,
+      "tokens_seen": 1804271616
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046105050505050506,
+      "loss": 1.3807,
+      "theoretical_loss": 3.460268823495631,
+      "tokens_seen": 1804337152
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000461030303030303,
+      "loss": 1.3098,
+      "theoretical_loss": 3.4602581313239273,
+      "tokens_seen": 1804402688
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046101010101010103,
+      "loss": 1.4589,
+      "theoretical_loss": 3.460247439649288,
+      "tokens_seen": 1804468224
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000460989898989899,
+      "loss": 1.2463,
+      "theoretical_loss": 3.4602367484716714,
+      "tokens_seen": 1804533760
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000460969696969697,
+      "loss": 1.5439,
+      "theoretical_loss": 3.4602260577910364,
+      "tokens_seen": 1804599296
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046094949494949496,
+      "loss": 1.3021,
+      "theoretical_loss": 3.4602153676073417,
+      "tokens_seen": 1804664832
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004609292929292929,
+      "loss": 1.4273,
+      "theoretical_loss": 3.4602046779205464,
+      "tokens_seen": 1804730368
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004609090909090909,
+      "loss": 1.587,
+      "theoretical_loss": 3.4601939887306097,
+      "tokens_seen": 1804795904
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046088888888888894,
+      "loss": 1.2694,
+      "theoretical_loss": 3.4601833000374898,
+      "tokens_seen": 1804861440
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004608686868686869,
+      "loss": 1.3032,
+      "theoretical_loss": 3.4601726118411458,
+      "tokens_seen": 1804926976
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046084848484848485,
+      "loss": 1.4727,
+      "theoretical_loss": 3.460161924141537,
+      "tokens_seen": 1804992512
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004608282828282828,
+      "loss": 1.4011,
+      "theoretical_loss": 3.4601512369386214,
+      "tokens_seen": 1805058048
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004608080808080808,
+      "loss": 1.2257,
+      "theoretical_loss": 3.4601405502323583,
+      "tokens_seen": 1805123584
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046078787878787883,
+      "loss": 1.373,
+      "theoretical_loss": 3.460129864022707,
+      "tokens_seen": 1805189120
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004607676767676768,
+      "loss": 1.2235,
+      "theoretical_loss": 3.460119178309626,
+      "tokens_seen": 1805254656
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046074747474747474,
+      "loss": 1.5574,
+      "theoretical_loss": 3.4601084930930743,
+      "tokens_seen": 1805320192
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004607272727272727,
+      "loss": 1.3258,
+      "theoretical_loss": 3.4600978083730105,
+      "tokens_seen": 1805385728
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046070707070707077,
+      "loss": 1.4703,
+      "theoretical_loss": 3.460087124149394,
+      "tokens_seen": 1805451264
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004606868686868687,
+      "loss": 1.2853,
+      "theoretical_loss": 3.4600764404221835,
+      "tokens_seen": 1805516800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004606666666666667,
+      "loss": 1.5211,
+      "theoretical_loss": 3.460065757191338,
+      "tokens_seen": 1805582336
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046064646464646464,
+      "loss": 1.5547,
+      "theoretical_loss": 3.460055074456816,
+      "tokens_seen": 1805647872
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004606262626262626,
+      "loss": 1.3416,
+      "theoretical_loss": 3.4600443922185766,
+      "tokens_seen": 1805713408
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/aligned_token_count": 0.40169844031333923,
+      "objective/train/avg_token_score": 0.0685867890715599,
+      "objective/train/docs_used": 1021470,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.3303582668304443,
+      "objective/train/original_loss": 4.088999271392822,
+      "objective/train/theoretical_loss": 3.4600337104765795,
+      "objective/train/tokens_used": 164638176,
+      "objective/train/unlikelihood_loss": 0.06166958063840866,
+      "theoretical_loss": 3.4600337104765795,
+      "tokens_seen": 1805778944
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046060606060606066,
+      "loss": 1.422,
+      "theoretical_loss": 3.4600337104765795,
+      "tokens_seen": 1805778944
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004605858585858586,
+      "loss": 1.3202,
+      "theoretical_loss": 3.4600230292307828,
+      "tokens_seen": 1805844480
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046056565656565657,
+      "loss": 1.2661,
+      "theoretical_loss": 3.4600123484811456,
+      "tokens_seen": 1805910016
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046054545454545453,
+      "loss": 1.5477,
+      "theoretical_loss": 3.460001668227627,
+      "tokens_seen": 1805975552
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046052525252525254,
+      "loss": 1.2371,
+      "theoretical_loss": 3.459990988470186,
+      "tokens_seen": 1806041088
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046050505050505055,
+      "loss": 1.3883,
+      "theoretical_loss": 3.459980309208781,
+      "tokens_seen": 1806106624
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004604848484848485,
+      "loss": 1.4474,
+      "theoretical_loss": 3.459969630443372,
+      "tokens_seen": 1806172160
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046046464646464646,
+      "loss": 1.4572,
+      "theoretical_loss": 3.459958952173917,
+      "tokens_seen": 1806237696
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004604444444444444,
+      "loss": 1.3687,
+      "theoretical_loss": 3.459948274400375,
+      "tokens_seen": 1806303232
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046042424242424243,
+      "loss": 1.5883,
+      "theoretical_loss": 3.459937597122706,
+      "tokens_seen": 1806368768
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046040404040404044,
+      "loss": 1.113,
+      "theoretical_loss": 3.459926920340868,
+      "tokens_seen": 1806434304
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004603838383838384,
+      "loss": 1.3119,
+      "theoretical_loss": 3.4599162440548197,
+      "tokens_seen": 1806499840
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046036363636363636,
+      "loss": 1.1213,
+      "theoretical_loss": 3.4599055682645217,
+      "tokens_seen": 1806565376
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046034343434343437,
+      "loss": 1.5036,
+      "theoretical_loss": 3.459894892969931,
+      "tokens_seen": 1806630912
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004603232323232323,
+      "loss": 1.4614,
+      "theoretical_loss": 3.459884218171008,
+      "tokens_seen": 1806696448
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046030303030303034,
+      "loss": 1.2264,
+      "theoretical_loss": 3.4598735438677117,
+      "tokens_seen": 1806761984
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004602828282828283,
+      "loss": 1.2862,
+      "theoretical_loss": 3.4598628700600003,
+      "tokens_seen": 1806827520
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046026262626262625,
+      "loss": 1.4969,
+      "theoretical_loss": 3.4598521967478337,
+      "tokens_seen": 1806893056
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046024242424242426,
+      "loss": 1.3143,
+      "theoretical_loss": 3.4598415239311695,
+      "tokens_seen": 1806958592
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004602222222222222,
+      "loss": 1.3717,
+      "theoretical_loss": 3.4598308516099685,
+      "tokens_seen": 1807024128
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046020202020202023,
+      "loss": 1.2754,
+      "theoretical_loss": 3.4598201797841885,
+      "tokens_seen": 1807089664
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004601818181818182,
+      "loss": 1.3765,
+      "theoretical_loss": 3.459809508453789,
+      "tokens_seen": 1807155200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004601616161616162,
+      "loss": 1.4214,
+      "theoretical_loss": 3.459798837618729,
+      "tokens_seen": 1807220736
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046014141414141415,
+      "loss": 1.2824,
+      "theoretical_loss": 3.4597881672789677,
+      "tokens_seen": 1807286272
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004601212121212121,
+      "loss": 1.2776,
+      "theoretical_loss": 3.4597774974344637,
+      "tokens_seen": 1807351808
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/aligned_token_count": 0.5522360801696777,
+      "objective/train/avg_token_score": 0.005671007093042135,
+      "objective/train/docs_used": 1022105,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.969338059425354,
+      "objective/train/original_loss": 3.9141592979431152,
+      "objective/train/theoretical_loss": 3.4597668280851765,
+      "objective/train/tokens_used": 166276576,
+      "objective/train/unlikelihood_loss": 0.07195361703634262,
+      "theoretical_loss": 3.4597668280851765,
+      "tokens_seen": 1807417344
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004601010101010101,
+      "loss": 1.4951,
+      "theoretical_loss": 3.4597668280851765,
+      "tokens_seen": 1807417344
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004600808080808081,
+      "loss": 1.4208,
+      "theoretical_loss": 3.4597561592310644,
+      "tokens_seen": 1807482880
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004600606060606061,
+      "loss": 1.4108,
+      "theoretical_loss": 3.4597454908720877,
+      "tokens_seen": 1807548416
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046004040404040405,
+      "loss": 1.5542,
+      "theoretical_loss": 3.459734823008205,
+      "tokens_seen": 1807613952
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000460020202020202,
+      "loss": 1.2047,
+      "theoretical_loss": 3.4597241556393747,
+      "tokens_seen": 1807679488
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00046,
+      "loss": 1.4753,
+      "theoretical_loss": 3.4597134887655567,
+      "tokens_seen": 1807745024
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000459979797979798,
+      "loss": 1.4563,
+      "theoretical_loss": 3.459702822386709,
+      "tokens_seen": 1807810560
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000459959595959596,
+      "loss": 1.4975,
+      "theoretical_loss": 3.4596921565027925,
+      "tokens_seen": 1807876096
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045993939393939394,
+      "loss": 1.5808,
+      "theoretical_loss": 3.4596814911137646,
+      "tokens_seen": 1807941632
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004599191919191919,
+      "loss": 1.3581,
+      "theoretical_loss": 3.459670826219585,
+      "tokens_seen": 1808007168
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004598989898989899,
+      "loss": 1.4058,
+      "theoretical_loss": 3.459660161820213,
+      "tokens_seen": 1808072704
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004598787878787879,
+      "loss": 1.4865,
+      "theoretical_loss": 3.459649497915608,
+      "tokens_seen": 1808138240
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004598585858585859,
+      "loss": 1.3384,
+      "theoretical_loss": 3.459638834505728,
+      "tokens_seen": 1808203776
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045983838383838383,
+      "loss": 1.2745,
+      "theoretical_loss": 3.4596281715905333,
+      "tokens_seen": 1808269312
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004598181818181818,
+      "loss": 1.2995,
+      "theoretical_loss": 3.4596175091699823,
+      "tokens_seen": 1808334848
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004597979797979798,
+      "loss": 1.2344,
+      "theoretical_loss": 3.459606847244034,
+      "tokens_seen": 1808400384
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004597777777777778,
+      "loss": 1.2524,
+      "theoretical_loss": 3.4595961858126483,
+      "tokens_seen": 1808465920
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045975757575757577,
+      "loss": 1.3612,
+      "theoretical_loss": 3.459585524875784,
+      "tokens_seen": 1808531456
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004597373737373737,
+      "loss": 1.3054,
+      "theoretical_loss": 3.4595748644334,
+      "tokens_seen": 1808596992
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004597171717171717,
+      "loss": 1.4338,
+      "theoretical_loss": 3.4595642044854555,
+      "tokens_seen": 1808662528
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045969696969696975,
+      "loss": 1.3573,
+      "theoretical_loss": 3.45955354503191,
+      "tokens_seen": 1808728064
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004596767676767677,
+      "loss": 1.2754,
+      "theoretical_loss": 3.4595428860727218,
+      "tokens_seen": 1808793600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045965656565656566,
+      "loss": 1.45,
+      "theoretical_loss": 3.4595322276078515,
+      "tokens_seen": 1808859136
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004596363636363636,
+      "loss": 1.3942,
+      "theoretical_loss": 3.459521569637257,
+      "tokens_seen": 1808924672
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004596161616161616,
+      "loss": 1.5969,
+      "theoretical_loss": 3.459510912160898,
+      "tokens_seen": 1808990208
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/aligned_token_count": 0.4491080045700073,
+      "objective/train/avg_token_score": 0.009337084367871284,
+      "objective/train/docs_used": 1022607,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.514532446861267,
+      "objective/train/original_loss": 3.918682336807251,
+      "objective/train/theoretical_loss": 3.4595002551787335,
+      "objective/train/tokens_used": 167914976,
+      "objective/train/unlikelihood_loss": 0.08269507437944412,
+      "theoretical_loss": 3.4595002551787335,
+      "tokens_seen": 1809055744
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045959595959595964,
+      "loss": 1.38,
+      "theoretical_loss": 3.4595002551787335,
+      "tokens_seen": 1809055744
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004595757575757576,
+      "loss": 1.4447,
+      "theoretical_loss": 3.4594895986907233,
+      "tokens_seen": 1809121280
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045955555555555555,
+      "loss": 1.2231,
+      "theoretical_loss": 3.4594789426968253,
+      "tokens_seen": 1809186816
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004595353535353535,
+      "loss": 1.5118,
+      "theoretical_loss": 3.4594682871969997,
+      "tokens_seen": 1809252352
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004595151515151515,
+      "loss": 1.3982,
+      "theoretical_loss": 3.4594576321912056,
+      "tokens_seen": 1809317888
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045949494949494953,
+      "loss": 1.3473,
+      "theoretical_loss": 3.459446977679402,
+      "tokens_seen": 1809383424
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004594747474747475,
+      "loss": 1.4469,
+      "theoretical_loss": 3.4594363236615484,
+      "tokens_seen": 1809448960
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045945454545454545,
+      "loss": 1.1953,
+      "theoretical_loss": 3.4594256701376036,
+      "tokens_seen": 1809514496
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004594343434343434,
+      "loss": 1.4947,
+      "theoretical_loss": 3.459415017107527,
+      "tokens_seen": 1809580032
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045941414141414147,
+      "loss": 1.1828,
+      "theoretical_loss": 3.4594043645712773,
+      "tokens_seen": 1809645568
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004593939393939394,
+      "loss": 1.5579,
+      "theoretical_loss": 3.459393712528815,
+      "tokens_seen": 1809711104
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004593737373737374,
+      "loss": 1.2093,
+      "theoretical_loss": 3.459383060980098,
+      "tokens_seen": 1809776640
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045935353535353534,
+      "loss": 1.3925,
+      "theoretical_loss": 3.459372409925087,
+      "tokens_seen": 1809842176
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045933333333333335,
+      "loss": 1.5784,
+      "theoretical_loss": 3.4593617593637394,
+      "tokens_seen": 1809907712
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045931313131313136,
+      "loss": 1.4518,
+      "theoretical_loss": 3.4593511092960156,
+      "tokens_seen": 1809973248
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004592929292929293,
+      "loss": 1.2988,
+      "theoretical_loss": 3.459340459721875,
+      "tokens_seen": 1810038784
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004592727272727273,
+      "loss": 1.4289,
+      "theoretical_loss": 3.459329810641276,
+      "tokens_seen": 1810104320
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045925252525252523,
+      "loss": 1.4281,
+      "theoretical_loss": 3.4593191620541783,
+      "tokens_seen": 1810169856
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045923232323232324,
+      "loss": 1.2985,
+      "theoretical_loss": 3.4593085139605417,
+      "tokens_seen": 1810235392
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045921212121212125,
+      "loss": 1.3204,
+      "theoretical_loss": 3.459297866360325,
+      "tokens_seen": 1810300928
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004591919191919192,
+      "loss": 1.4494,
+      "theoretical_loss": 3.459287219253487,
+      "tokens_seen": 1810366464
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045917171717171717,
+      "loss": 1.5598,
+      "theoretical_loss": 3.459276572639988,
+      "tokens_seen": 1810432000
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004591515151515152,
+      "loss": 1.3901,
+      "theoretical_loss": 3.459265926519786,
+      "tokens_seen": 1810497536
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045913131313131313,
+      "loss": 1.3048,
+      "theoretical_loss": 3.459255280892841,
+      "tokens_seen": 1810563072
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045911111111111115,
+      "loss": 1.4061,
+      "theoretical_loss": 3.459244635759113,
+      "tokens_seen": 1810628608
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/aligned_token_count": 0.2624022364616394,
+      "objective/train/avg_token_score": 0.027153268456459045,
+      "objective/train/docs_used": 1024104,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.0158097743988037,
+      "objective/train/original_loss": 5.203606128692627,
+      "objective/train/theoretical_loss": 3.459233991118561,
+      "objective/train/tokens_used": 169553376,
+      "objective/train/unlikelihood_loss": 0.043073952198028564,
+      "theoretical_loss": 3.459233991118561,
+      "tokens_seen": 1810694144
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004590909090909091,
+      "loss": 1.26,
+      "theoretical_loss": 3.459233991118561,
+      "tokens_seen": 1810694144
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045907070707070706,
+      "loss": 1.2706,
+      "theoretical_loss": 3.4592233469711426,
+      "tokens_seen": 1810759680
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045905050505050507,
+      "loss": 1.0934,
+      "theoretical_loss": 3.459212703316819,
+      "tokens_seen": 1810825216
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045903030303030303,
+      "loss": 1.3351,
+      "theoretical_loss": 3.4592020601555493,
+      "tokens_seen": 1810890752
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045901010101010104,
+      "loss": 1.2675,
+      "theoretical_loss": 3.4591914174872915,
+      "tokens_seen": 1810956288
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000458989898989899,
+      "loss": 1.3725,
+      "theoretical_loss": 3.4591807753120065,
+      "tokens_seen": 1811021824
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000458969696969697,
+      "loss": 1.4893,
+      "theoretical_loss": 3.4591701336296534,
+      "tokens_seen": 1811087360
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045894949494949496,
+      "loss": 1.347,
+      "theoretical_loss": 3.4591594924401905,
+      "tokens_seen": 1811152896
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004589292929292929,
+      "loss": 1.4972,
+      "theoretical_loss": 3.459148851743578,
+      "tokens_seen": 1811218432
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045890909090909093,
+      "loss": 1.2867,
+      "theoretical_loss": 3.459138211539775,
+      "tokens_seen": 1811283968
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004588888888888889,
+      "loss": 1.4704,
+      "theoretical_loss": 3.4591275718287404,
+      "tokens_seen": 1811349504
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004588686868686869,
+      "loss": 1.27,
+      "theoretical_loss": 3.459116932610435,
+      "tokens_seen": 1811415040
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045884848484848486,
+      "loss": 1.3998,
+      "theoretical_loss": 3.4591062938848163,
+      "tokens_seen": 1811480576
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004588282828282828,
+      "loss": 1.4274,
+      "theoretical_loss": 3.4590956556518444,
+      "tokens_seen": 1811546112
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004588080808080808,
+      "loss": 1.5243,
+      "theoretical_loss": 3.4590850179114794,
+      "tokens_seen": 1811611648
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045878787878787883,
+      "loss": 1.3508,
+      "theoretical_loss": 3.45907438066368,
+      "tokens_seen": 1811677184
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004587676767676768,
+      "loss": 1.4852,
+      "theoretical_loss": 3.4590637439084055,
+      "tokens_seen": 1811742720
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045874747474747475,
+      "loss": 1.3903,
+      "theoretical_loss": 3.459053107645615,
+      "tokens_seen": 1811808256
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004587272727272727,
+      "loss": 1.5327,
+      "theoretical_loss": 3.459042471875269,
+      "tokens_seen": 1811873792
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004587070707070707,
+      "loss": 1.202,
+      "theoretical_loss": 3.4590318365973256,
+      "tokens_seen": 1811939328
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045868686868686873,
+      "loss": 1.2374,
+      "theoretical_loss": 3.459021201811745,
+      "tokens_seen": 1812004864
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004586666666666667,
+      "loss": 1.3289,
+      "theoretical_loss": 3.4590105675184866,
+      "tokens_seen": 1812070400
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045864646464646464,
+      "loss": 1.4467,
+      "theoretical_loss": 3.458999933717509,
+      "tokens_seen": 1812135936
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004586262626262626,
+      "loss": 1.1531,
+      "theoretical_loss": 3.4589893004087733,
+      "tokens_seen": 1812201472
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045860606060606066,
+      "loss": 1.4899,
+      "theoretical_loss": 3.4589786675922367,
+      "tokens_seen": 1812267008
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/aligned_token_count": 0.4100073277950287,
+      "objective/train/avg_token_score": 0.012066290713846684,
+      "objective/train/docs_used": 1024786,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.3427298069000244,
+      "objective/train/original_loss": 4.287503242492676,
+      "objective/train/theoretical_loss": 3.4589680352678602,
+      "objective/train/tokens_used": 171191776,
+      "objective/train/unlikelihood_loss": 0.05709630250930786,
+      "theoretical_loss": 3.4589680352678602,
+      "tokens_seen": 1812332544
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004585858585858586,
+      "loss": 1.2515,
+      "theoretical_loss": 3.4589680352678602,
+      "tokens_seen": 1812332544
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004585656565656566,
+      "loss": 1.2303,
+      "theoretical_loss": 3.4589574034356025,
+      "tokens_seen": 1812398080
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045854545454545453,
+      "loss": 1.6304,
+      "theoretical_loss": 3.4589467720954232,
+      "tokens_seen": 1812463616
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004585252525252525,
+      "loss": 1.3892,
+      "theoretical_loss": 3.4589361412472828,
+      "tokens_seen": 1812529152
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045850505050505056,
+      "loss": 1.4854,
+      "theoretical_loss": 3.4589255108911385,
+      "tokens_seen": 1812594688
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004584848484848485,
+      "loss": 1.5415,
+      "theoretical_loss": 3.458914881026952,
+      "tokens_seen": 1812660224
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045846464646464647,
+      "loss": 1.4956,
+      "theoretical_loss": 3.458904251654681,
+      "tokens_seen": 1812725760
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004584444444444444,
+      "loss": 1.3248,
+      "theoretical_loss": 3.458893622774286,
+      "tokens_seen": 1812791296
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004584242424242424,
+      "loss": 1.3537,
+      "theoretical_loss": 3.4588829943857258,
+      "tokens_seen": 1812856832
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045840404040404045,
+      "loss": 1.4755,
+      "theoretical_loss": 3.4588723664889605,
+      "tokens_seen": 1812922368
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004583838383838384,
+      "loss": 1.2218,
+      "theoretical_loss": 3.4588617390839493,
+      "tokens_seen": 1812987904
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045836363636363636,
+      "loss": 1.3251,
+      "theoretical_loss": 3.4588511121706516,
+      "tokens_seen": 1813053440
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004583434343434343,
+      "loss": 1.4598,
+      "theoretical_loss": 3.458840485749027,
+      "tokens_seen": 1813118976
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045832323232323233,
+      "loss": 1.6122,
+      "theoretical_loss": 3.4588298598190352,
+      "tokens_seen": 1813184512
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045830303030303034,
+      "loss": 1.1923,
+      "theoretical_loss": 3.458819234380635,
+      "tokens_seen": 1813250048
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004582828282828283,
+      "loss": 1.3857,
+      "theoretical_loss": 3.4588086094337864,
+      "tokens_seen": 1813315584
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045826262626262626,
+      "loss": 1.4029,
+      "theoretical_loss": 3.4587979849784487,
+      "tokens_seen": 1813381120
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004582424242424242,
+      "loss": 1.3769,
+      "theoretical_loss": 3.458787361014582,
+      "tokens_seen": 1813446656
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004582222222222222,
+      "loss": 1.3538,
+      "theoretical_loss": 3.4587767375421445,
+      "tokens_seen": 1813512192
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045820202020202023,
+      "loss": 1.2557,
+      "theoretical_loss": 3.458766114561097,
+      "tokens_seen": 1813577728
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004581818181818182,
+      "loss": 1.4583,
+      "theoretical_loss": 3.458755492071398,
+      "tokens_seen": 1813643264
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045816161616161615,
+      "loss": 1.3099,
+      "theoretical_loss": 3.458744870073008,
+      "tokens_seen": 1813708800
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045814141414141416,
+      "loss": 1.2544,
+      "theoretical_loss": 3.458734248565886,
+      "tokens_seen": 1813774336
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004581212121212121,
+      "loss": 1.461,
+      "theoretical_loss": 3.4587236275499915,
+      "tokens_seen": 1813839872
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004581010101010101,
+      "loss": 1.1234,
+      "theoretical_loss": 3.458713007025284,
+      "tokens_seen": 1813905408
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/aligned_token_count": 0.4995112419128418,
+      "objective/train/avg_token_score": 0.0071546295657753944,
+      "objective/train/docs_used": 1026014,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.7556525468826294,
+      "objective/train/original_loss": 3.839704990386963,
+      "objective/train/theoretical_loss": 3.4587023869917237,
+      "objective/train/tokens_used": 172830176,
+      "objective/train/unlikelihood_loss": 0.09379331767559052,
+      "theoretical_loss": 3.4587023869917237,
+      "tokens_seen": 1813970944
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004580808080808081,
+      "loss": 1.4608,
+      "theoretical_loss": 3.4587023869917237,
+      "tokens_seen": 1813970944
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045806060606060604,
+      "loss": 1.3605,
+      "theoretical_loss": 3.458691767449269,
+      "tokens_seen": 1814036480
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045804040404040405,
+      "loss": 1.1711,
+      "theoretical_loss": 3.45868114839788,
+      "tokens_seen": 1814102016
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000458020202020202,
+      "loss": 1.3351,
+      "theoretical_loss": 3.4586705298375167,
+      "tokens_seen": 1814167552
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000458,
+      "loss": 1.5054,
+      "theoretical_loss": 3.4586599117681383,
+      "tokens_seen": 1814233088
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000457979797979798,
+      "loss": 1.6297,
+      "theoretical_loss": 3.458649294189704,
+      "tokens_seen": 1814298624
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.000457959595959596,
+      "loss": 1.4436,
+      "theoretical_loss": 3.458638677102174,
+      "tokens_seen": 1814364160
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045793939393939394,
+      "loss": 1.3552,
+      "theoretical_loss": 3.4586280605055073,
+      "tokens_seen": 1814429696
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004579191919191919,
+      "loss": 1.5167,
+      "theoretical_loss": 3.458617444399664,
+      "tokens_seen": 1814495232
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004578989898989899,
+      "loss": 1.4367,
+      "theoretical_loss": 3.4586068287846032,
+      "tokens_seen": 1814560768
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045787878787878787,
+      "loss": 1.4934,
+      "theoretical_loss": 3.458596213660285,
+      "tokens_seen": 1814626304
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004578585858585859,
+      "loss": 1.3668,
+      "theoretical_loss": 3.4585855990266685,
+      "tokens_seen": 1814691840
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045783838383838384,
+      "loss": 1.2387,
+      "theoretical_loss": 3.4585749848837133,
+      "tokens_seen": 1814757376
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004578181818181818,
+      "loss": 1.2531,
+      "theoretical_loss": 3.45856437123138,
+      "tokens_seen": 1814822912
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004577979797979798,
+      "loss": 1.4417,
+      "theoretical_loss": 3.4585537580696264,
+      "tokens_seen": 1814888448
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004577777777777778,
+      "loss": 1.6412,
+      "theoretical_loss": 3.458543145398414,
+      "tokens_seen": 1814953984
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045775757575757577,
+      "loss": 1.6142,
+      "theoretical_loss": 3.458532533217701,
+      "tokens_seen": 1815019520
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045773737373737373,
+      "loss": 1.5157,
+      "theoretical_loss": 3.4585219215274474,
+      "tokens_seen": 1815085056
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045771717171717174,
+      "loss": 1.43,
+      "theoretical_loss": 3.458511310327613,
+      "tokens_seen": 1815150592
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004576969696969697,
+      "loss": 1.2891,
+      "theoretical_loss": 3.458500699618158,
+      "tokens_seen": 1815216128
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004576767676767677,
+      "loss": 1.4889,
+      "theoretical_loss": 3.458490089399041,
+      "tokens_seen": 1815281664
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045765656565656567,
+      "loss": 1.5082,
+      "theoretical_loss": 3.4584794796702223,
+      "tokens_seen": 1815347200
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004576363636363636,
+      "loss": 1.3993,
+      "theoretical_loss": 3.458468870431661,
+      "tokens_seen": 1815412736
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045761616161616163,
+      "loss": 1.4451,
+      "theoretical_loss": 3.4584582616833175,
+      "tokens_seen": 1815478272
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045759595959595964,
+      "loss": 1.3518,
+      "theoretical_loss": 3.458447653425151,
+      "tokens_seen": 1815543808
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/aligned_token_count": 0.34109237790107727,
+      "objective/train/avg_token_score": 0.014077051542699337,
+      "objective/train/docs_used": 1026828,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.3032065629959106,
+      "objective/train/original_loss": 4.5359601974487305,
+      "objective/train/theoretical_loss": 3.4584370456571207,
+      "objective/train/tokens_used": 174468576,
+      "objective/train/unlikelihood_loss": 0.057071033865213394,
+      "theoretical_loss": 3.4584370456571207,
+      "tokens_seen": 1815609344
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004575757575757576,
+      "loss": 1.4086,
+      "theoretical_loss": 3.4584370456571207,
+      "tokens_seen": 1815609344
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045755555555555556,
+      "loss": 1.3255,
+      "theoretical_loss": 3.4584264383791874,
+      "tokens_seen": 1815674880
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004575353535353535,
+      "loss": 1.5165,
+      "theoretical_loss": 3.4584158315913096,
+      "tokens_seen": 1815740416
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004575151515151515,
+      "loss": 1.4522,
+      "theoretical_loss": 3.458405225293448,
+      "tokens_seen": 1815805952
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045749494949494954,
+      "loss": 1.4464,
+      "theoretical_loss": 3.458394619485561,
+      "tokens_seen": 1815871488
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004574747474747475,
+      "loss": 1.495,
+      "theoretical_loss": 3.4583840141676094,
+      "tokens_seen": 1815937024
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045745454545454545,
+      "loss": 1.6631,
+      "theoretical_loss": 3.458373409339553,
+      "tokens_seen": 1816002560
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004574343434343434,
+      "loss": 1.4861,
+      "theoretical_loss": 3.4583628050013506,
+      "tokens_seen": 1816068096
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004574141414141415,
+      "loss": 1.312,
+      "theoretical_loss": 3.4583522011529624,
+      "tokens_seen": 1816133632
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045739393939393943,
+      "loss": 1.3567,
+      "theoretical_loss": 3.458341597794348,
+      "tokens_seen": 1816199168
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004573737373737374,
+      "loss": 1.4321,
+      "theoretical_loss": 3.4583309949254675,
+      "tokens_seen": 1816264704
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045735353535353534,
+      "loss": 1.3804,
+      "theoretical_loss": 3.4583203925462795,
+      "tokens_seen": 1816330240
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004573333333333333,
+      "loss": 1.292,
+      "theoretical_loss": 3.4583097906567453,
+      "tokens_seen": 1816395776
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045731313131313137,
+      "loss": 1.3965,
+      "theoretical_loss": 3.458299189256823,
+      "tokens_seen": 1816461312
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004572929292929293,
+      "loss": 1.532,
+      "theoretical_loss": 3.458288588346474,
+      "tokens_seen": 1816526848
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004572727272727273,
+      "loss": 1.5313,
+      "theoretical_loss": 3.4582779879256567,
+      "tokens_seen": 1816592384
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045725252525252524,
+      "loss": 1.314,
+      "theoretical_loss": 3.458267387994331,
+      "tokens_seen": 1816657920
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045723232323232325,
+      "loss": 1.4601,
+      "theoretical_loss": 3.458256788552457,
+      "tokens_seen": 1816723456
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045721212121212126,
+      "loss": 1.169,
+      "theoretical_loss": 3.4582461895999947,
+      "tokens_seen": 1816788992
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004571919191919192,
+      "loss": 1.7361,
+      "theoretical_loss": 3.458235591136903,
+      "tokens_seen": 1816854528
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045717171717171717,
+      "loss": 1.4227,
+      "theoretical_loss": 3.458224993163143,
+      "tokens_seen": 1816920064
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045715151515151513,
+      "loss": 1.4055,
+      "theoretical_loss": 3.458214395678673,
+      "tokens_seen": 1816985600
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045713131313131314,
+      "loss": 1.37,
+      "theoretical_loss": 3.4582037986834537,
+      "tokens_seen": 1817051136
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.00045711111111111115,
+      "loss": 1.6846,
+      "theoretical_loss": 3.458193202177444,
+      "tokens_seen": 1817116672
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 0.0004570909090909091,
+      "loss": 1.2868,
+      "theoretical_loss": 3.4581826061606047,
+      "tokens_seen": 1817182208
+    },
+    {
+      "epoch": 0.09,
+      "objective/train/aligned_token_count": 0.3228861093521118,
+      "objective/train/avg_token_score": 0.038064464926719666,
+      "objective/train/docs_used": 1028029,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.0878517627716064,
+      "objective/train/original_loss": 4.760405540466309,
+      "objective/train/theoretical_loss": 3.458172010632895,
+      "objective/train/tokens_used": 176106976,
+      "objective/train/unlikelihood_loss": 0.04580136016011238,
+      "theoretical_loss": 3.458172010632895,
+      "tokens_seen": 1817247744
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045707070707070706,
+      "loss": 1.159,
+      "theoretical_loss": 3.458172010632895,
+      "tokens_seen": 1817247744
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004570505050505051,
+      "loss": 1.4579,
+      "theoretical_loss": 3.458161415594275,
+      "tokens_seen": 1817313280
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045703030303030303,
+      "loss": 1.3157,
+      "theoretical_loss": 3.458150821044704,
+      "tokens_seen": 1817378816
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045701010101010104,
+      "loss": 1.324,
+      "theoretical_loss": 3.458140226984142,
+      "tokens_seen": 1817444352
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000456989898989899,
+      "loss": 1.2537,
+      "theoretical_loss": 3.458129633412549,
+      "tokens_seen": 1817509888
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045696969696969696,
+      "loss": 1.6337,
+      "theoretical_loss": 3.4581190403298847,
+      "tokens_seen": 1817575424
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045694949494949497,
+      "loss": 1.3909,
+      "theoretical_loss": 3.458108447736109,
+      "tokens_seen": 1817640960
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004569292929292929,
+      "loss": 1.3148,
+      "theoretical_loss": 3.4580978556311814,
+      "tokens_seen": 1817706496
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045690909090909094,
+      "loss": 1.2169,
+      "theoretical_loss": 3.458087264015062,
+      "tokens_seen": 1817772032
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004568888888888889,
+      "loss": 1.5658,
+      "theoretical_loss": 3.45807667288771,
+      "tokens_seen": 1817837568
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004568686868686869,
+      "loss": 1.5285,
+      "theoretical_loss": 3.4580660822490863,
+      "tokens_seen": 1817903104
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045684848484848486,
+      "loss": 1.2645,
+      "theoretical_loss": 3.45805549209915,
+      "tokens_seen": 1817968640
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004568282828282828,
+      "loss": 1.5466,
+      "theoretical_loss": 3.4580449024378614,
+      "tokens_seen": 1818034176
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045680808080808083,
+      "loss": 1.4763,
+      "theoretical_loss": 3.4580343132651796,
+      "tokens_seen": 1818099712
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004567878787878788,
+      "loss": 1.3013,
+      "theoretical_loss": 3.458023724581065,
+      "tokens_seen": 1818165248
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004567676767676768,
+      "loss": 1.4691,
+      "theoretical_loss": 3.4580131363854782,
+      "tokens_seen": 1818230784
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045674747474747475,
+      "loss": 1.3829,
+      "theoretical_loss": 3.458002548678377,
+      "tokens_seen": 1818296320
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004567272727272727,
+      "loss": 1.4288,
+      "theoretical_loss": 3.457991961459723,
+      "tokens_seen": 1818361856
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004567070707070707,
+      "loss": 1.7387,
+      "theoretical_loss": 3.4579813747294756,
+      "tokens_seen": 1818427392
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045668686868686873,
+      "loss": 1.2347,
+      "theoretical_loss": 3.4579707884875948,
+      "tokens_seen": 1818492928
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004566666666666667,
+      "loss": 1.3851,
+      "theoretical_loss": 3.4579602027340393,
+      "tokens_seen": 1818558464
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045664646464646465,
+      "loss": 1.4735,
+      "theoretical_loss": 3.457949617468771,
+      "tokens_seen": 1818624000
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004566262626262626,
+      "loss": 1.4051,
+      "theoretical_loss": 3.457939032691748,
+      "tokens_seen": 1818689536
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004566060606060606,
+      "loss": 1.4462,
+      "theoretical_loss": 3.4579284484029316,
+      "tokens_seen": 1818755072
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004565858585858586,
+      "loss": 1.2984,
+      "theoretical_loss": 3.45791786460228,
+      "tokens_seen": 1818820608
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/aligned_token_count": 0.464809387922287,
+      "objective/train/avg_token_score": 0.016029013320803642,
+      "objective/train/docs_used": 1028792,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.6073720455169678,
+      "objective/train/original_loss": 3.9546051025390625,
+      "objective/train/theoretical_loss": 3.4579072812897547,
+      "objective/train/tokens_used": 177745376,
+      "objective/train/unlikelihood_loss": 0.07868336141109467,
+      "theoretical_loss": 3.4579072812897547,
+      "tokens_seen": 1818886144
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004565656565656566,
+      "loss": 1.5138,
+      "theoretical_loss": 3.4579072812897547,
+      "tokens_seen": 1818886144
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045654545454545454,
+      "loss": 1.4139,
+      "theoretical_loss": 3.457896698465315,
+      "tokens_seen": 1818951680
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004565252525252525,
+      "loss": 1.5099,
+      "theoretical_loss": 3.457886116128921,
+      "tokens_seen": 1819017216
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004565050505050505,
+      "loss": 1.3184,
+      "theoretical_loss": 3.4578755342805323,
+      "tokens_seen": 1819082752
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004564848484848485,
+      "loss": 1.4564,
+      "theoretical_loss": 3.457864952920109,
+      "tokens_seen": 1819148288
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004564646464646465,
+      "loss": 1.4435,
+      "theoretical_loss": 3.4578543720476103,
+      "tokens_seen": 1819213824
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045644444444444443,
+      "loss": 1.6601,
+      "theoretical_loss": 3.4578437916629974,
+      "tokens_seen": 1819279360
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004564242424242424,
+      "loss": 1.3769,
+      "theoretical_loss": 3.4578332117662294,
+      "tokens_seen": 1819344896
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045640404040404045,
+      "loss": 1.4961,
+      "theoretical_loss": 3.4578226323572663,
+      "tokens_seen": 1819410432
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004563838383838384,
+      "loss": 1.2399,
+      "theoretical_loss": 3.4578120534360686,
+      "tokens_seen": 1819475968
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045636363636363637,
+      "loss": 1.5018,
+      "theoretical_loss": 3.4578014750025954,
+      "tokens_seen": 1819541504
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004563434343434343,
+      "loss": 1.4262,
+      "theoretical_loss": 3.457790897056807,
+      "tokens_seen": 1819607040
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004563232323232323,
+      "loss": 1.4682,
+      "theoretical_loss": 3.4577803195986636,
+      "tokens_seen": 1819672576
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045630303030303035,
+      "loss": 1.5494,
+      "theoretical_loss": 3.457769742628125,
+      "tokens_seen": 1819738112
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004562828282828283,
+      "loss": 1.4778,
+      "theoretical_loss": 3.4577591661451517,
+      "tokens_seen": 1819803648
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045626262626262626,
+      "loss": 1.459,
+      "theoretical_loss": 3.457748590149703,
+      "tokens_seen": 1819869184
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004562424242424242,
+      "loss": 1.5946,
+      "theoretical_loss": 3.457738014641738,
+      "tokens_seen": 1819934720
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045622222222222223,
+      "loss": 1.3618,
+      "theoretical_loss": 3.4577274396212183,
+      "tokens_seen": 1820000256
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045620202020202024,
+      "loss": 1.2889,
+      "theoretical_loss": 3.457716865088103,
+      "tokens_seen": 1820065792
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004561818181818182,
+      "loss": 1.3204,
+      "theoretical_loss": 3.457706291042353,
+      "tokens_seen": 1820131328
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045616161616161615,
+      "loss": 1.4486,
+      "theoretical_loss": 3.4576957174839267,
+      "tokens_seen": 1820196864
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004561414141414141,
+      "loss": 1.2579,
+      "theoretical_loss": 3.4576851444127854,
+      "tokens_seen": 1820262400
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004561212121212121,
+      "loss": 1.2631,
+      "theoretical_loss": 3.457674571828889,
+      "tokens_seen": 1820327936
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045610101010101013,
+      "loss": 1.2376,
+      "theoretical_loss": 3.457663999732197,
+      "tokens_seen": 1820393472
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004560808080808081,
+      "loss": 1.3863,
+      "theoretical_loss": 3.4576534281226694,
+      "tokens_seen": 1820459008
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/aligned_token_count": 0.5623167157173157,
+      "objective/train/avg_token_score": 0.014118832536041737,
+      "objective/train/docs_used": 1029550,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.7937812805175781,
+      "objective/train/original_loss": 3.694382429122925,
+      "objective/train/theoretical_loss": 3.457642857000267,
+      "objective/train/tokens_used": 179383776,
+      "objective/train/unlikelihood_loss": 0.07253962755203247,
+      "theoretical_loss": 3.457642857000267,
+      "tokens_seen": 1820524544
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045606060606060605,
+      "loss": 1.5616,
+      "theoretical_loss": 3.457642857000267,
+      "tokens_seen": 1820524544
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045604040404040406,
+      "loss": 1.4623,
+      "theoretical_loss": 3.457632286364949,
+      "tokens_seen": 1820590080
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045602020202020207,
+      "loss": 1.5794,
+      "theoretical_loss": 3.4576217162166754,
+      "tokens_seen": 1820655616
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000456,
+      "loss": 1.4721,
+      "theoretical_loss": 3.457611146555407,
+      "tokens_seen": 1820721152
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000455979797979798,
+      "loss": 1.5718,
+      "theoretical_loss": 3.457600577381103,
+      "tokens_seen": 1820786688
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045595959595959594,
+      "loss": 1.3938,
+      "theoretical_loss": 3.4575900086937237,
+      "tokens_seen": 1820852224
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045593939393939395,
+      "loss": 1.5038,
+      "theoretical_loss": 3.4575794404932294,
+      "tokens_seen": 1820917760
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045591919191919196,
+      "loss": 1.4119,
+      "theoretical_loss": 3.4575688727795804,
+      "tokens_seen": 1820983296
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004558989898989899,
+      "loss": 1.3345,
+      "theoretical_loss": 3.4575583055527357,
+      "tokens_seen": 1821048832
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004558787878787879,
+      "loss": 1.2874,
+      "theoretical_loss": 3.4575477388126563,
+      "tokens_seen": 1821114368
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004558585858585859,
+      "loss": 1.1633,
+      "theoretical_loss": 3.457537172559302,
+      "tokens_seen": 1821179904
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045583838383838384,
+      "loss": 1.4255,
+      "theoretical_loss": 3.4575266067926327,
+      "tokens_seen": 1821245440
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045581818181818185,
+      "loss": 1.4251,
+      "theoretical_loss": 3.4575160415126085,
+      "tokens_seen": 1821310976
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004557979797979798,
+      "loss": 1.4595,
+      "theoretical_loss": 3.4575054767191897,
+      "tokens_seen": 1821376512
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045577777777777777,
+      "loss": 1.4112,
+      "theoretical_loss": 3.4574949124123364,
+      "tokens_seen": 1821442048
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004557575757575758,
+      "loss": 1.3731,
+      "theoretical_loss": 3.457484348592008,
+      "tokens_seen": 1821507584
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045573737373737374,
+      "loss": 1.6005,
+      "theoretical_loss": 3.4574737852581654,
+      "tokens_seen": 1821573120
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045571717171717175,
+      "loss": 1.4437,
+      "theoretical_loss": 3.4574632224107686,
+      "tokens_seen": 1821638656
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004556969696969697,
+      "loss": 1.6006,
+      "theoretical_loss": 3.4574526600497775,
+      "tokens_seen": 1821704192
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004556767676767677,
+      "loss": 1.3622,
+      "theoretical_loss": 3.457442098175152,
+      "tokens_seen": 1821769728
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045565656565656567,
+      "loss": 1.4977,
+      "theoretical_loss": 3.457431536786853,
+      "tokens_seen": 1821835264
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045563636363636363,
+      "loss": 1.5852,
+      "theoretical_loss": 3.4574209758848395,
+      "tokens_seen": 1821900800
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045561616161616164,
+      "loss": 1.3857,
+      "theoretical_loss": 3.457410415469072,
+      "tokens_seen": 1821966336
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004555959595959596,
+      "loss": 1.5078,
+      "theoretical_loss": 3.4573998555395113,
+      "tokens_seen": 1822031872
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004555757575757576,
+      "loss": 1.4179,
+      "theoretical_loss": 3.4573892960961166,
+      "tokens_seen": 1822097408
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/aligned_token_count": 0.4111681282520294,
+      "objective/train/avg_token_score": 0.017115775495767593,
+      "objective/train/docs_used": 1030259,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.2357131242752075,
+      "objective/train/original_loss": 3.788912296295166,
+      "objective/train/theoretical_loss": 3.457378737138849,
+      "objective/train/tokens_used": 181022176,
+      "objective/train/unlikelihood_loss": 0.0951184332370758,
+      "theoretical_loss": 3.457378737138849,
+      "tokens_seen": 1822162944
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045555555555555556,
+      "loss": 1.2408,
+      "theoretical_loss": 3.457378737138849,
+      "tokens_seen": 1822162944
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004555353535353535,
+      "loss": 1.3708,
+      "theoretical_loss": 3.4573681786676675,
+      "tokens_seen": 1822228480
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045551515151515153,
+      "loss": 1.5096,
+      "theoretical_loss": 3.457357620682533,
+      "tokens_seen": 1822294016
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045549494949494954,
+      "loss": 1.3569,
+      "theoretical_loss": 3.4573470631834056,
+      "tokens_seen": 1822359552
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004554747474747475,
+      "loss": 1.3497,
+      "theoretical_loss": 3.457336506170245,
+      "tokens_seen": 1822425088
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045545454545454546,
+      "loss": 1.5165,
+      "theoretical_loss": 3.457325949643012,
+      "tokens_seen": 1822490624
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004554343434343434,
+      "loss": 1.5752,
+      "theoretical_loss": 3.4573153936016663,
+      "tokens_seen": 1822556160
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004554141414141414,
+      "loss": 1.2614,
+      "theoretical_loss": 3.4573048380461686,
+      "tokens_seen": 1822621696
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045539393939393944,
+      "loss": 1.4106,
+      "theoretical_loss": 3.4572942829764783,
+      "tokens_seen": 1822687232
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004553737373737374,
+      "loss": 1.4145,
+      "theoretical_loss": 3.4572837283925564,
+      "tokens_seen": 1822752768
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045535353535353535,
+      "loss": 1.4457,
+      "theoretical_loss": 3.457273174294362,
+      "tokens_seen": 1822818304
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004553333333333333,
+      "loss": 1.418,
+      "theoretical_loss": 3.457262620681856,
+      "tokens_seen": 1822883840
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045531313131313137,
+      "loss": 1.3548,
+      "theoretical_loss": 3.4572520675549985,
+      "tokens_seen": 1822949376
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045529292929292933,
+      "loss": 1.4232,
+      "theoretical_loss": 3.45724151491375,
+      "tokens_seen": 1823014912
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004552727272727273,
+      "loss": 1.5722,
+      "theoretical_loss": 3.4572309627580706,
+      "tokens_seen": 1823080448
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045525252525252524,
+      "loss": 1.6458,
+      "theoretical_loss": 3.4572204110879197,
+      "tokens_seen": 1823145984
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004552323232323232,
+      "loss": 1.352,
+      "theoretical_loss": 3.4572098599032586,
+      "tokens_seen": 1823211520
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045521212121212126,
+      "loss": 1.6786,
+      "theoretical_loss": 3.4571993092040465,
+      "tokens_seen": 1823277056
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004551919191919192,
+      "loss": 1.6506,
+      "theoretical_loss": 3.4571887589902444,
+      "tokens_seen": 1823342592
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004551717171717172,
+      "loss": 1.2894,
+      "theoretical_loss": 3.457178209261812,
+      "tokens_seen": 1823408128
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045515151515151513,
+      "loss": 1.3039,
+      "theoretical_loss": 3.457167660018711,
+      "tokens_seen": 1823473664
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004551313131313131,
+      "loss": 1.1383,
+      "theoretical_loss": 3.4571571112608988,
+      "tokens_seen": 1823539200
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045511111111111116,
+      "loss": 1.3979,
+      "theoretical_loss": 3.4571465629883384,
+      "tokens_seen": 1823604736
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004550909090909091,
+      "loss": 1.3313,
+      "theoretical_loss": 3.4571360152009882,
+      "tokens_seen": 1823670272
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045507070707070707,
+      "loss": 1.5223,
+      "theoretical_loss": 3.4571254678988095,
+      "tokens_seen": 1823735808
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/aligned_token_count": 0.38483625650405884,
+      "objective/train/avg_token_score": 0.019340477883815765,
+      "objective/train/docs_used": 1031478,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.2517280578613281,
+      "objective/train/original_loss": 4.015500068664551,
+      "objective/train/theoretical_loss": 3.4571149210817618,
+      "objective/train/tokens_used": 182660576,
+      "objective/train/unlikelihood_loss": 0.09212837368249893,
+      "theoretical_loss": 3.4571149210817618,
+      "tokens_seen": 1823801344
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045505050505050503,
+      "loss": 1.3643,
+      "theoretical_loss": 3.4571149210817618,
+      "tokens_seen": 1823801344
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045503030303030304,
+      "loss": 1.4704,
+      "theoretical_loss": 3.4571043747498065,
+      "tokens_seen": 1823866880
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045501010101010105,
+      "loss": 1.5384,
+      "theoretical_loss": 3.4570938289029023,
+      "tokens_seen": 1823932416
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000454989898989899,
+      "loss": 1.518,
+      "theoretical_loss": 3.457083283541011,
+      "tokens_seen": 1823997952
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045496969696969696,
+      "loss": 1.1422,
+      "theoretical_loss": 3.4570727386640914,
+      "tokens_seen": 1824063488
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004549494949494949,
+      "loss": 1.3168,
+      "theoretical_loss": 3.4570621942721047,
+      "tokens_seen": 1824129024
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045492929292929293,
+      "loss": 1.3456,
+      "theoretical_loss": 3.4570516503650115,
+      "tokens_seen": 1824194560
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045490909090909094,
+      "loss": 1.4976,
+      "theoretical_loss": 3.4570411069427713,
+      "tokens_seen": 1824260096
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004548888888888889,
+      "loss": 1.4185,
+      "theoretical_loss": 3.4570305640053443,
+      "tokens_seen": 1824325632
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045486868686868686,
+      "loss": 1.3554,
+      "theoretical_loss": 3.457020021552691,
+      "tokens_seen": 1824391168
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045484848484848487,
+      "loss": 1.4911,
+      "theoretical_loss": 3.4570094795847726,
+      "tokens_seen": 1824456704
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004548282828282828,
+      "loss": 1.3135,
+      "theoretical_loss": 3.456998938101548,
+      "tokens_seen": 1824522240
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045480808080808083,
+      "loss": 1.0882,
+      "theoretical_loss": 3.456988397102979,
+      "tokens_seen": 1824587776
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004547878787878788,
+      "loss": 1.3685,
+      "theoretical_loss": 3.4569778565890243,
+      "tokens_seen": 1824653312
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045476767676767675,
+      "loss": 1.4268,
+      "theoretical_loss": 3.456967316559645,
+      "tokens_seen": 1824718848
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045474747474747476,
+      "loss": 1.5668,
+      "theoretical_loss": 3.4569567770148018,
+      "tokens_seen": 1824784384
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004547272727272727,
+      "loss": 1.6636,
+      "theoretical_loss": 3.4569462379544547,
+      "tokens_seen": 1824849920
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045470707070707073,
+      "loss": 1.4786,
+      "theoretical_loss": 3.4569356993785636,
+      "tokens_seen": 1824915456
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004546868686868687,
+      "loss": 1.3914,
+      "theoretical_loss": 3.456925161287089,
+      "tokens_seen": 1824980992
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004546666666666667,
+      "loss": 1.2798,
+      "theoretical_loss": 3.456914623679992,
+      "tokens_seen": 1825046528
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045464646464646465,
+      "loss": 1.4608,
+      "theoretical_loss": 3.4569040865572322,
+      "tokens_seen": 1825112064
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004546262626262626,
+      "loss": 1.6782,
+      "theoretical_loss": 3.4568935499187696,
+      "tokens_seen": 1825177600
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004546060606060606,
+      "loss": 1.4233,
+      "theoretical_loss": 3.456883013764566,
+      "tokens_seen": 1825243136
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004545858585858586,
+      "loss": 1.5035,
+      "theoretical_loss": 3.4568724780945796,
+      "tokens_seen": 1825308672
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004545656565656566,
+      "loss": 1.4682,
+      "theoretical_loss": 3.456861942908773,
+      "tokens_seen": 1825374208
+    },
+    {
+      "debugging/Self-BLEU-5": 0.383412367211192,
+      "debugging/distinct-1-grams": 0.7526443544193546,
+      "debugging/distinct-2-grams": 0.9328595808568966,
+      "debugging/entropy-1-grams": 5.657827037788948,
+      "debugging/entropy-2-grams": 6.397750202501998,
+      "debugging/length": 484.77777777777777,
+      "debugging/num_segments": 9,
+      "epoch": 0.1,
+      "objective/train/aligned_token_count": 0.5272482633590698,
+      "objective/train/avg_token_score": 0.014473862946033478,
+      "objective/train/docs_used": 1032081,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.6364436149597168,
+      "objective/train/original_loss": 3.784334897994995,
+      "objective/train/theoretical_loss": 3.456851408207105,
+      "objective/train/tokens_used": 184298976,
+      "objective/train/unlikelihood_loss": 0.08270136266946793,
+      "theoretical_loss": 3.456851408207105,
+      "tokens_seen": 1825439744
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045454545454545455,
+      "loss": 1.618,
+      "theoretical_loss": 3.456851408207105,
+      "tokens_seen": 1825439744
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004545252525252525,
+      "loss": 1.2274,
+      "theoretical_loss": 3.456840873989537,
+      "tokens_seen": 1825505280
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004545050505050505,
+      "loss": 1.3231,
+      "theoretical_loss": 3.4568303402560288,
+      "tokens_seen": 1825570816
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004544848484848485,
+      "loss": 1.3608,
+      "theoretical_loss": 3.4568198070065406,
+      "tokens_seen": 1825636352
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004544646464646465,
+      "loss": 1.4591,
+      "theoretical_loss": 3.4568092742410332,
+      "tokens_seen": 1825701888
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045444444444444444,
+      "loss": 1.2171,
+      "theoretical_loss": 3.456798741959467,
+      "tokens_seen": 1825767424
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004544242424242424,
+      "loss": 1.4365,
+      "theoretical_loss": 3.456788210161802,
+      "tokens_seen": 1825832960
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004544040404040404,
+      "loss": 1.4114,
+      "theoretical_loss": 3.456777678847999,
+      "tokens_seen": 1825898496
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004543838383838384,
+      "loss": 1.3937,
+      "theoretical_loss": 3.4567671480180184,
+      "tokens_seen": 1825964032
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004543636363636364,
+      "loss": 1.4272,
+      "theoretical_loss": 3.4567566176718203,
+      "tokens_seen": 1826029568
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045434343434343433,
+      "loss": 1.5479,
+      "theoretical_loss": 3.456746087809365,
+      "tokens_seen": 1826095104
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004543232323232323,
+      "loss": 1.1661,
+      "theoretical_loss": 3.456735558430614,
+      "tokens_seen": 1826160640
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045430303030303035,
+      "loss": 1.1125,
+      "theoretical_loss": 3.4567250295355265,
+      "tokens_seen": 1826226176
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004542828282828283,
+      "loss": 1.4769,
+      "theoretical_loss": 3.456714501124063,
+      "tokens_seen": 1826291712
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045426262626262627,
+      "loss": 1.3936,
+      "theoretical_loss": 3.4567039731961846,
+      "tokens_seen": 1826357248
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004542424242424242,
+      "loss": 1.2966,
+      "theoretical_loss": 3.4566934457518514,
+      "tokens_seen": 1826422784
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045422222222222223,
+      "loss": 1.341,
+      "theoretical_loss": 3.456682918791024,
+      "tokens_seen": 1826488320
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045420202020202025,
+      "loss": 1.4049,
+      "theoretical_loss": 3.4566723923136626,
+      "tokens_seen": 1826553856
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004541818181818182,
+      "loss": 1.4241,
+      "theoretical_loss": 3.4566618663197275,
+      "tokens_seen": 1826619392
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045416161616161616,
+      "loss": 1.3875,
+      "theoretical_loss": 3.45665134080918,
+      "tokens_seen": 1826684928
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004541414141414141,
+      "loss": 1.5876,
+      "theoretical_loss": 3.4566408157819795,
+      "tokens_seen": 1826750464
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004541212121212122,
+      "loss": 1.3572,
+      "theoretical_loss": 3.456630291238087,
+      "tokens_seen": 1826816000
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045410101010101014,
+      "loss": 1.5581,
+      "theoretical_loss": 3.4566197671774628,
+      "tokens_seen": 1826881536
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004540808080808081,
+      "loss": 1.4737,
+      "theoretical_loss": 3.456609243600068,
+      "tokens_seen": 1826947072
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045406060606060605,
+      "loss": 1.428,
+      "theoretical_loss": 3.456598720505862,
+      "tokens_seen": 1827012608
+    },
+    {
+      "epoch": 0.1,
+      "objective/train/aligned_token_count": 0.3259408473968506,
+      "objective/train/avg_token_score": 0.02392873726785183,
+      "objective/train/docs_used": 1033277,
+      "objective/train/instantaneous_batch_size": 16,
+      "objective/train/instantaneous_microbatch_size": 16384,
+      "objective/train/likelihood_loss": 1.0870252847671509,
+      "objective/train/original_loss": 3.9227612018585205,
+      "objective/train/theoretical_loss": 3.4565881978948063,
+      "objective/train/tokens_used": 185937376,
+      "objective/train/unlikelihood_loss": 0.08083102852106094,
+      "theoretical_loss": 3.4565881978948063,
+      "tokens_seen": 1827078144
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000454040404040404,
+      "loss": 1.1697,
+      "theoretical_loss": 3.4565881978948063,
+      "tokens_seen": 1827078144
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004540202020202021,
+      "loss": 1.3098,
+      "theoretical_loss": 3.4565776757668605,
+      "tokens_seen": 1827143680
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045400000000000003,
+      "loss": 1.3512,
+      "theoretical_loss": 3.456567154121986,
+      "tokens_seen": 1827209216
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.000453979797979798,
+      "loss": 1.438,
+      "theoretical_loss": 3.4565566329601425,
+      "tokens_seen": 1827274752
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045395959595959594,
+      "loss": 1.6695,
+      "theoretical_loss": 3.4565461122812904,
+      "tokens_seen": 1827340288
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045393939393939396,
+      "loss": 1.1057,
+      "theoretical_loss": 3.4565355920853915,
+      "tokens_seen": 1827405824
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045391919191919197,
+      "loss": 1.5515,
+      "theoretical_loss": 3.456525072372405,
+      "tokens_seen": 1827471360
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004538989898989899,
+      "loss": 1.2735,
+      "theoretical_loss": 3.456514553142292,
+      "tokens_seen": 1827536896
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.0004538787878787879,
+      "loss": 1.3733,
+      "theoretical_loss": 3.456504034395013,
+      "tokens_seen": 1827602432
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045385858585858584,
+      "loss": 1.3649,
+      "theoretical_loss": 3.4564935161305277,
+      "tokens_seen": 1827667968
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00045383838383838385,
+      "loss": 1.7118,
+      "theoretical_loss": 3.4564829983487977,
+      "tokens_seen": 1827733504
+    }
+  ],
+  "max_steps": 25000,
+  "num_train_epochs": 9223372036854775807,
+  "total_flos": 8.478403854336e+16,
+  "trial_name": null,
+  "trial_params": null
+}