diff --git "a/checkpoint-7605/trainer_state.json" "b/checkpoint-7605/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-7605/trainer_state.json" @@ -0,0 +1,57832 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.3042, + "global_step": 7605, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 2e-06, + "loss": 2.3586, + "theoretical_loss": 3.484799908140819, + "tokens_seen": 1661665280 + }, + { + "epoch": 0.0, + "learning_rate": 4e-06, + "loss": 2.2339, + "theoretical_loss": 3.4847880270596066, + "tokens_seen": 1661730816 + }, + { + "epoch": 0.0, + "learning_rate": 6e-06, + "loss": 2.2016, + "theoretical_loss": 3.4847761465781497, + "tokens_seen": 1661796352 + }, + { + "epoch": 0.0, + "learning_rate": 8e-06, + "loss": 2.2196, + "theoretical_loss": 3.484764266696394, + "tokens_seen": 1661861888 + }, + { + "epoch": 0.0, + "learning_rate": 1e-05, + "loss": 2.1452, + "theoretical_loss": 3.4847523874142867, + "tokens_seen": 1661927424 + }, + { + "epoch": 0.0, + "learning_rate": 1.2e-05, + "loss": 2.1924, + "theoretical_loss": 3.4847405087317727, + "tokens_seen": 1661992960 + }, + { + "epoch": 0.0, + "learning_rate": 1.4e-05, + "loss": 2.3474, + "theoretical_loss": 3.4847286306487986, + "tokens_seen": 1662058496 + }, + { + "epoch": 0.0, + "learning_rate": 1.6e-05, + "loss": 2.3439, + "theoretical_loss": 3.48471675316531, + "tokens_seen": 1662124032 + }, + { + "epoch": 0.0, + "learning_rate": 1.8e-05, + "loss": 2.1693, + "theoretical_loss": 3.4847048762812536, + "tokens_seen": 1662189568 + }, + { + "epoch": 0.0, + "learning_rate": 2e-05, + "loss": 2.0245, + "theoretical_loss": 3.4846929999965757, + "tokens_seen": 1662255104 + }, + { + "epoch": 0.0, + "learning_rate": 2.2e-05, + "loss": 2.071, + "theoretical_loss": 3.484681124311222, + "tokens_seen": 1662320640 + }, + { + "epoch": 0.0, + "learning_rate": 2.4e-05, + "loss": 1.9012, + "theoretical_loss": 3.4846692492251385, + "tokens_seen": 1662386176 + }, + { + "epoch": 0.0, + "learning_rate": 2.6e-05, + "loss": 1.8901, + "theoretical_loss": 3.484657374738272, + "tokens_seen": 1662451712 + }, + { + "epoch": 0.0, + "learning_rate": 2.8e-05, + "loss": 1.9415, + "theoretical_loss": 3.4846455008505677, + "tokens_seen": 1662517248 + }, + { + "epoch": 0.0, + "learning_rate": 3e-05, + "loss": 1.9999, + "theoretical_loss": 3.484633627561972, + "tokens_seen": 1662582784 + }, + { + "epoch": 0.0, + "learning_rate": 3.2e-05, + "loss": 1.9898, + "theoretical_loss": 3.484621754872432, + "tokens_seen": 1662648320 + }, + { + "epoch": 0.0, + "learning_rate": 3.4000000000000007e-05, + "loss": 1.6089, + "theoretical_loss": 3.4846098827818928, + "tokens_seen": 1662713856 + }, + { + "epoch": 0.0, + "learning_rate": 3.6e-05, + "loss": 1.7149, + "theoretical_loss": 3.4845980112903003, + "tokens_seen": 1662779392 + }, + { + "epoch": 0.0, + "learning_rate": 3.8e-05, + "loss": 1.6944, + "theoretical_loss": 3.484586140397602, + "tokens_seen": 1662844928 + }, + { + "epoch": 0.0, + "learning_rate": 4e-05, + "loss": 1.4261, + "theoretical_loss": 3.484574270103743, + "tokens_seen": 1662910464 + }, + { + "epoch": 0.0, + "learning_rate": 4.2000000000000004e-05, + "loss": 1.7173, + "theoretical_loss": 3.484562400408671, + "tokens_seen": 1662976000 + }, + { + "epoch": 0.0, + "learning_rate": 4.4e-05, + "loss": 1.5526, + "theoretical_loss": 3.4845505313123297, + "tokens_seen": 1663041536 + }, + { + "epoch": 0.0, + "learning_rate": 4.6e-05, + "loss": 1.7101, + "theoretical_loss": 3.4845386628146677, + "tokens_seen": 1663107072 + }, + { + "epoch": 0.0, + "learning_rate": 4.8e-05, + "loss": 1.5717, + "theoretical_loss": 3.4845267949156296, + "tokens_seen": 1663172608 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.4392717480659485, + "objective/train/avg_token_score": 0.027796979993581772, + "objective/train/docs_used": 941634, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5729044675827026, + "objective/train/original_loss": 3.6452524662017822, + "objective/train/theoretical_loss": 3.4845149276151623, + "objective/train/tokens_used": 22097376, + "objective/train/unlikelihood_loss": 0.09146079421043396, + "theoretical_loss": 3.4845149276151623, + "tokens_seen": 1663238144 + }, + { + "epoch": 0.0, + "learning_rate": 5e-05, + "loss": 1.6885, + "theoretical_loss": 3.4845149276151623, + "tokens_seen": 1663238144 + }, + { + "epoch": 0.0, + "learning_rate": 5.2e-05, + "loss": 1.4219, + "theoretical_loss": 3.484503060913212, + "tokens_seen": 1663303680 + }, + { + "epoch": 0.0, + "learning_rate": 5.4e-05, + "loss": 1.5592, + "theoretical_loss": 3.484491194809725, + "tokens_seen": 1663369216 + }, + { + "epoch": 0.0, + "learning_rate": 5.6e-05, + "loss": 1.1071, + "theoretical_loss": 3.484479329304647, + "tokens_seen": 1663434752 + }, + { + "epoch": 0.0, + "learning_rate": 5.800000000000001e-05, + "loss": 1.4801, + "theoretical_loss": 3.484467464397925, + "tokens_seen": 1663500288 + }, + { + "epoch": 0.0, + "learning_rate": 6e-05, + "loss": 1.6332, + "theoretical_loss": 3.484455600089505, + "tokens_seen": 1663565824 + }, + { + "epoch": 0.0, + "learning_rate": 6.2e-05, + "loss": 1.4538, + "theoretical_loss": 3.4844437363793332, + "tokens_seen": 1663631360 + }, + { + "epoch": 0.0, + "learning_rate": 6.4e-05, + "loss": 1.5288, + "theoretical_loss": 3.4844318732673556, + "tokens_seen": 1663696896 + }, + { + "epoch": 0.0, + "learning_rate": 6.6e-05, + "loss": 1.4459, + "theoretical_loss": 3.484420010753519, + "tokens_seen": 1663762432 + }, + { + "epoch": 0.0, + "learning_rate": 6.800000000000001e-05, + "loss": 1.4075, + "theoretical_loss": 3.484408148837769, + "tokens_seen": 1663827968 + }, + { + "epoch": 0.0, + "learning_rate": 7.000000000000001e-05, + "loss": 1.2731, + "theoretical_loss": 3.4843962875200525, + "tokens_seen": 1663893504 + }, + { + "epoch": 0.0, + "learning_rate": 7.2e-05, + "loss": 1.2299, + "theoretical_loss": 3.484384426800316, + "tokens_seen": 1663959040 + }, + { + "epoch": 0.0, + "learning_rate": 7.4e-05, + "loss": 1.6132, + "theoretical_loss": 3.4843725666785046, + "tokens_seen": 1664024576 + }, + { + "epoch": 0.0, + "learning_rate": 7.6e-05, + "loss": 1.639, + "theoretical_loss": 3.484360707154566, + "tokens_seen": 1664090112 + }, + { + "epoch": 0.0, + "learning_rate": 7.8e-05, + "loss": 1.3998, + "theoretical_loss": 3.484348848228446, + "tokens_seen": 1664155648 + }, + { + "epoch": 0.0, + "learning_rate": 8e-05, + "loss": 1.6466, + "theoretical_loss": 3.4843369899000907, + "tokens_seen": 1664221184 + }, + { + "epoch": 0.0, + "learning_rate": 8.2e-05, + "loss": 1.4191, + "theoretical_loss": 3.484325132169446, + "tokens_seen": 1664286720 + }, + { + "epoch": 0.0, + "learning_rate": 8.400000000000001e-05, + "loss": 1.3433, + "theoretical_loss": 3.4843132750364596, + "tokens_seen": 1664352256 + }, + { + "epoch": 0.0, + "learning_rate": 8.599999999999999e-05, + "loss": 1.6113, + "theoretical_loss": 3.4843014185010768, + "tokens_seen": 1664417792 + }, + { + "epoch": 0.0, + "learning_rate": 8.8e-05, + "loss": 1.5214, + "theoretical_loss": 3.4842895625632444, + "tokens_seen": 1664483328 + }, + { + "epoch": 0.0, + "learning_rate": 8.999999999999999e-05, + "loss": 1.5353, + "theoretical_loss": 3.4842777072229083, + "tokens_seen": 1664548864 + }, + { + "epoch": 0.0, + "learning_rate": 9.2e-05, + "loss": 1.4214, + "theoretical_loss": 3.484265852480015, + "tokens_seen": 1664614400 + }, + { + "epoch": 0.0, + "learning_rate": 9.400000000000001e-05, + "loss": 1.4677, + "theoretical_loss": 3.4842539983345118, + "tokens_seen": 1664679936 + }, + { + "epoch": 0.0, + "learning_rate": 9.6e-05, + "loss": 1.4979, + "theoretical_loss": 3.484242144786344, + "tokens_seen": 1664745472 + }, + { + "epoch": 0.0, + "learning_rate": 9.800000000000001e-05, + "loss": 1.5655, + "theoretical_loss": 3.4842302918354577, + "tokens_seen": 1664811008 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.4052419364452362, + "objective/train/avg_token_score": 0.023708028718829155, + "objective/train/docs_used": 943199, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3439230918884277, + "objective/train/original_loss": 3.491624593734741, + "objective/train/theoretical_loss": 3.4842184394818, + "objective/train/tokens_used": 23735776, + "objective/train/unlikelihood_loss": 0.11052291095256805, + "theoretical_loss": 3.4842184394818, + "tokens_seen": 1664876544 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001, + "loss": 1.2824, + "theoretical_loss": 3.4842184394818, + "tokens_seen": 1664876544 + }, + { + "epoch": 0.0, + "learning_rate": 0.000102, + "loss": 1.3515, + "theoretical_loss": 3.4842065877253177, + "tokens_seen": 1664942080 + }, + { + "epoch": 0.0, + "learning_rate": 0.000104, + "loss": 1.327, + "theoretical_loss": 3.4841947365659562, + "tokens_seen": 1665007616 + }, + { + "epoch": 0.0, + "learning_rate": 0.000106, + "loss": 1.4169, + "theoretical_loss": 3.4841828860036625, + "tokens_seen": 1665073152 + }, + { + "epoch": 0.0, + "learning_rate": 0.000108, + "loss": 1.4514, + "theoretical_loss": 3.484171036038383, + "tokens_seen": 1665138688 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011, + "loss": 1.374, + "theoretical_loss": 3.4841591866700643, + "tokens_seen": 1665204224 + }, + { + "epoch": 0.0, + "learning_rate": 0.000112, + "loss": 1.5037, + "theoretical_loss": 3.4841473378986523, + "tokens_seen": 1665269760 + }, + { + "epoch": 0.0, + "learning_rate": 0.000114, + "loss": 1.5909, + "theoretical_loss": 3.4841354897240935, + "tokens_seen": 1665335296 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011600000000000001, + "loss": 1.3882, + "theoretical_loss": 3.4841236421463346, + "tokens_seen": 1665400832 + }, + { + "epoch": 0.0, + "learning_rate": 0.000118, + "loss": 1.4278, + "theoretical_loss": 3.4841117951653224, + "tokens_seen": 1665466368 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012, + "loss": 1.2305, + "theoretical_loss": 3.4840999487810027, + "tokens_seen": 1665531904 + }, + { + "epoch": 0.0, + "learning_rate": 0.000122, + "loss": 1.3588, + "theoretical_loss": 3.4840881029933226, + "tokens_seen": 1665597440 + }, + { + "epoch": 0.0, + "learning_rate": 0.000124, + "loss": 1.1805, + "theoretical_loss": 3.484076257802228, + "tokens_seen": 1665662976 + }, + { + "epoch": 0.0, + "learning_rate": 0.000126, + "loss": 1.4335, + "theoretical_loss": 3.4840644132076655, + "tokens_seen": 1665728512 + }, + { + "epoch": 0.0, + "learning_rate": 0.000128, + "loss": 1.5079, + "theoretical_loss": 3.484052569209582, + "tokens_seen": 1665794048 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013000000000000002, + "loss": 1.319, + "theoretical_loss": 3.4840407258079233, + "tokens_seen": 1665859584 + }, + { + "epoch": 0.0, + "learning_rate": 0.000132, + "loss": 1.4067, + "theoretical_loss": 3.4840288830026367, + "tokens_seen": 1665925120 + }, + { + "epoch": 0.0, + "learning_rate": 0.000134, + "loss": 1.2983, + "theoretical_loss": 3.484017040793668, + "tokens_seen": 1665990656 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013600000000000003, + "loss": 1.2986, + "theoretical_loss": 3.4840051991809644, + "tokens_seen": 1666056192 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013800000000000002, + "loss": 1.2998, + "theoretical_loss": 3.4839933581644718, + "tokens_seen": 1666121728 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014000000000000001, + "loss": 1.5612, + "theoretical_loss": 3.4839815177441373, + "tokens_seen": 1666187264 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014199999999999998, + "loss": 1.3289, + "theoretical_loss": 3.4839696779199065, + "tokens_seen": 1666252800 + }, + { + "epoch": 0.0, + "learning_rate": 0.000144, + "loss": 1.4071, + "theoretical_loss": 3.483957838691727, + "tokens_seen": 1666318336 + }, + { + "epoch": 0.0, + "learning_rate": 0.000146, + "loss": 1.1692, + "theoretical_loss": 3.483946000059545, + "tokens_seen": 1666383872 + }, + { + "epoch": 0.0, + "learning_rate": 0.000148, + "loss": 1.4058, + "theoretical_loss": 3.483934162023307, + "tokens_seen": 1666449408 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.3277125954627991, + "objective/train/avg_token_score": 0.01894918829202652, + "objective/train/docs_used": 943856, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0503660440444946, + "objective/train/original_loss": 3.4445393085479736, + "objective/train/theoretical_loss": 3.4839223245829594, + "objective/train/tokens_used": 25374176, + "objective/train/unlikelihood_loss": 0.0962013378739357, + "theoretical_loss": 3.4839223245829594, + "tokens_seen": 1666514944 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015, + "loss": 1.1905, + "theoretical_loss": 3.4839223245829594, + "tokens_seen": 1666514944 + }, + { + "epoch": 0.0, + "learning_rate": 0.000152, + "loss": 1.5246, + "theoretical_loss": 3.4839104877384486, + "tokens_seen": 1666580480 + }, + { + "epoch": 0.0, + "learning_rate": 0.000154, + "loss": 1.3905, + "theoretical_loss": 3.483898651489722, + "tokens_seen": 1666646016 + }, + { + "epoch": 0.0, + "learning_rate": 0.000156, + "loss": 1.4391, + "theoretical_loss": 3.4838868158367253, + "tokens_seen": 1666711552 + }, + { + "epoch": 0.0, + "learning_rate": 0.000158, + "loss": 1.2598, + "theoretical_loss": 3.4838749807794063, + "tokens_seen": 1666777088 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016, + "loss": 1.4413, + "theoretical_loss": 3.4838631463177103, + "tokens_seen": 1666842624 + }, + { + "epoch": 0.0, + "learning_rate": 0.000162, + "loss": 1.5131, + "theoretical_loss": 3.483851312451584, + "tokens_seen": 1666908160 + }, + { + "epoch": 0.0, + "learning_rate": 0.000164, + "loss": 1.333, + "theoretical_loss": 3.4838394791809746, + "tokens_seen": 1666973696 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016600000000000002, + "loss": 1.6326, + "theoretical_loss": 3.4838276465058287, + "tokens_seen": 1667039232 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016800000000000002, + "loss": 1.5681, + "theoretical_loss": 3.4838158144260927, + "tokens_seen": 1667104768 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017, + "loss": 1.1809, + "theoretical_loss": 3.483803982941714, + "tokens_seen": 1667170304 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017199999999999998, + "loss": 1.2542, + "theoretical_loss": 3.4837921520526374, + "tokens_seen": 1667235840 + }, + { + "epoch": 0.0, + "learning_rate": 0.000174, + "loss": 1.3605, + "theoretical_loss": 3.483780321758811, + "tokens_seen": 1667301376 + }, + { + "epoch": 0.0, + "learning_rate": 0.000176, + "loss": 1.6095, + "theoretical_loss": 3.4837684920601815, + "tokens_seen": 1667366912 + }, + { + "epoch": 0.0, + "learning_rate": 0.000178, + "loss": 1.4505, + "theoretical_loss": 3.483756662956695, + "tokens_seen": 1667432448 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017999999999999998, + "loss": 1.2638, + "theoretical_loss": 3.483744834448298, + "tokens_seen": 1667497984 + }, + { + "epoch": 0.0, + "learning_rate": 0.000182, + "loss": 1.4185, + "theoretical_loss": 3.483733006534938, + "tokens_seen": 1667563520 + }, + { + "epoch": 0.0, + "learning_rate": 0.000184, + "loss": 1.2676, + "theoretical_loss": 3.483721179216561, + "tokens_seen": 1667629056 + }, + { + "epoch": 0.0, + "learning_rate": 0.000186, + "loss": 1.7296, + "theoretical_loss": 3.4837093524931144, + "tokens_seen": 1667694592 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018800000000000002, + "loss": 1.6747, + "theoretical_loss": 3.4836975263645438, + "tokens_seen": 1667760128 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019, + "loss": 1.463, + "theoretical_loss": 3.483685700830797, + "tokens_seen": 1667825664 + }, + { + "epoch": 0.0, + "learning_rate": 0.000192, + "loss": 1.2903, + "theoretical_loss": 3.48367387589182, + "tokens_seen": 1667891200 + }, + { + "epoch": 0.0, + "learning_rate": 0.000194, + "loss": 1.3391, + "theoretical_loss": 3.48366205154756, + "tokens_seen": 1667956736 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019600000000000002, + "loss": 1.6197, + "theoretical_loss": 3.4836502277979626, + "tokens_seen": 1668022272 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019800000000000002, + "loss": 1.3482, + "theoretical_loss": 3.4836384046429765, + "tokens_seen": 1668087808 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.3819648027420044, + "objective/train/avg_token_score": 0.015858473256230354, + "objective/train/docs_used": 945126, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2602568864822388, + "objective/train/original_loss": 3.6047937870025635, + "objective/train/theoretical_loss": 3.4836265820825467, + "objective/train/tokens_used": 27012576, + "objective/train/unlikelihood_loss": 0.07870878279209137, + "theoretical_loss": 3.4836265820825467, + "tokens_seen": 1668153344 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002, + "loss": 1.3153, + "theoretical_loss": 3.4836265820825467, + "tokens_seen": 1668153344 + }, + { + "epoch": 0.0, + "learning_rate": 0.000202, + "loss": 1.313, + "theoretical_loss": 3.483614760116621, + "tokens_seen": 1668218880 + }, + { + "epoch": 0.0, + "learning_rate": 0.000204, + "loss": 1.4786, + "theoretical_loss": 3.4836029387451455, + "tokens_seen": 1668284416 + }, + { + "epoch": 0.0, + "learning_rate": 0.000206, + "loss": 1.4841, + "theoretical_loss": 3.483591117968067, + "tokens_seen": 1668349952 + }, + { + "epoch": 0.0, + "learning_rate": 0.000208, + "loss": 1.4115, + "theoretical_loss": 3.483579297785333, + "tokens_seen": 1668415488 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021, + "loss": 1.5871, + "theoretical_loss": 3.483567478196889, + "tokens_seen": 1668481024 + }, + { + "epoch": 0.0, + "learning_rate": 0.000212, + "loss": 1.5155, + "theoretical_loss": 3.483555659202683, + "tokens_seen": 1668546560 + }, + { + "epoch": 0.0, + "learning_rate": 0.000214, + "loss": 1.6009, + "theoretical_loss": 3.4835438408026613, + "tokens_seen": 1668612096 + }, + { + "epoch": 0.0, + "learning_rate": 0.000216, + "loss": 1.1647, + "theoretical_loss": 3.483532022996771, + "tokens_seen": 1668677632 + }, + { + "epoch": 0.0, + "learning_rate": 0.000218, + "loss": 1.3228, + "theoretical_loss": 3.483520205784958, + "tokens_seen": 1668743168 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022, + "loss": 1.4557, + "theoretical_loss": 3.48350838916717, + "tokens_seen": 1668808704 + }, + { + "epoch": 0.0, + "learning_rate": 0.000222, + "loss": 1.3735, + "theoretical_loss": 3.4834965731433534, + "tokens_seen": 1668874240 + }, + { + "epoch": 0.0, + "learning_rate": 0.000224, + "loss": 1.3874, + "theoretical_loss": 3.4834847577134553, + "tokens_seen": 1668939776 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022600000000000002, + "loss": 1.3996, + "theoretical_loss": 3.4834729428774223, + "tokens_seen": 1669005312 + }, + { + "epoch": 0.0, + "learning_rate": 0.000228, + "loss": 1.2483, + "theoretical_loss": 3.4834611286352013, + "tokens_seen": 1669070848 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023, + "loss": 1.5426, + "theoretical_loss": 3.4834493149867387, + "tokens_seen": 1669136384 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023200000000000003, + "loss": 1.1371, + "theoretical_loss": 3.483437501931982, + "tokens_seen": 1669201920 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023400000000000002, + "loss": 1.5659, + "theoretical_loss": 3.483425689470878, + "tokens_seen": 1669267456 + }, + { + "epoch": 0.0, + "learning_rate": 0.000236, + "loss": 1.5264, + "theoretical_loss": 3.4834138776033736, + "tokens_seen": 1669332992 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023799999999999998, + "loss": 1.3228, + "theoretical_loss": 3.483402066329415, + "tokens_seen": 1669398528 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024, + "loss": 1.4733, + "theoretical_loss": 3.48339025564895, + "tokens_seen": 1669464064 + }, + { + "epoch": 0.0, + "learning_rate": 0.000242, + "loss": 1.329, + "theoretical_loss": 3.4833784455619248, + "tokens_seen": 1669529600 + }, + { + "epoch": 0.0, + "learning_rate": 0.000244, + "loss": 1.4319, + "theoretical_loss": 3.4833666360682867, + "tokens_seen": 1669595136 + }, + { + "epoch": 0.0, + "learning_rate": 0.000246, + "loss": 1.4501, + "theoretical_loss": 3.4833548271679824, + "tokens_seen": 1669660672 + }, + { + "epoch": 0.0, + "learning_rate": 0.000248, + "loss": 1.3319, + "theoretical_loss": 3.4833430188609587, + "tokens_seen": 1669726208 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.45992180705070496, + "objective/train/avg_token_score": 0.009118808433413506, + "objective/train/docs_used": 945699, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6401410102844238, + "objective/train/original_loss": 4.021496772766113, + "objective/train/theoretical_loss": 3.4833312111471626, + "objective/train/tokens_used": 28650976, + "objective/train/unlikelihood_loss": 0.06683876365423203, + "theoretical_loss": 3.4833312111471626, + "tokens_seen": 1669791744 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025, + "loss": 1.6621, + "theoretical_loss": 3.4833312111471626, + "tokens_seen": 1669791744 + }, + { + "epoch": 0.01, + "learning_rate": 0.000252, + "loss": 1.361, + "theoretical_loss": 3.483319404026541, + "tokens_seen": 1669857280 + }, + { + "epoch": 0.01, + "learning_rate": 0.000254, + "loss": 1.1887, + "theoretical_loss": 3.4833075974990404, + "tokens_seen": 1669922816 + }, + { + "epoch": 0.01, + "learning_rate": 0.000256, + "loss": 1.4238, + "theoretical_loss": 3.4832957915646086, + "tokens_seen": 1669988352 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025800000000000004, + "loss": 1.4582, + "theoretical_loss": 3.4832839862231926, + "tokens_seen": 1670053888 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026000000000000003, + "loss": 1.509, + "theoretical_loss": 3.4832721814747383, + "tokens_seen": 1670119424 + }, + { + "epoch": 0.01, + "learning_rate": 0.000262, + "loss": 1.6246, + "theoretical_loss": 3.4832603773191932, + "tokens_seen": 1670184960 + }, + { + "epoch": 0.01, + "learning_rate": 0.000264, + "loss": 1.3161, + "theoretical_loss": 3.4832485737565047, + "tokens_seen": 1670250496 + }, + { + "epoch": 0.01, + "learning_rate": 0.000266, + "loss": 1.6053, + "theoretical_loss": 3.483236770786619, + "tokens_seen": 1670316032 + }, + { + "epoch": 0.01, + "learning_rate": 0.000268, + "loss": 1.2865, + "theoretical_loss": 3.4832249684094836, + "tokens_seen": 1670381568 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027, + "loss": 1.3504, + "theoretical_loss": 3.4832131666250454, + "tokens_seen": 1670447104 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027200000000000005, + "loss": 1.4324, + "theoretical_loss": 3.483201365433251, + "tokens_seen": 1670512640 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027400000000000005, + "loss": 1.478, + "theoretical_loss": 3.4831895648340474, + "tokens_seen": 1670578176 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027600000000000004, + "loss": 1.3201, + "theoretical_loss": 3.4831777648273823, + "tokens_seen": 1670643712 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027800000000000004, + "loss": 1.3004, + "theoretical_loss": 3.4831659654132023, + "tokens_seen": 1670709248 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028000000000000003, + "loss": 1.2886, + "theoretical_loss": 3.4831541665914543, + "tokens_seen": 1670774784 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028199999999999997, + "loss": 1.4002, + "theoretical_loss": 3.4831423683620852, + "tokens_seen": 1670840320 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028399999999999996, + "loss": 1.0925, + "theoretical_loss": 3.4831305707250424, + "tokens_seen": 1670905856 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028599999999999996, + "loss": 1.594, + "theoretical_loss": 3.483118773680273, + "tokens_seen": 1670971392 + }, + { + "epoch": 0.01, + "learning_rate": 0.000288, + "loss": 1.5114, + "theoretical_loss": 3.4831069772277234, + "tokens_seen": 1671036928 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029, + "loss": 1.4329, + "theoretical_loss": 3.483095181367341, + "tokens_seen": 1671102464 + }, + { + "epoch": 0.01, + "learning_rate": 0.000292, + "loss": 1.5268, + "theoretical_loss": 3.4830833860990733, + "tokens_seen": 1671168000 + }, + { + "epoch": 0.01, + "learning_rate": 0.000294, + "loss": 1.3822, + "theoretical_loss": 3.4830715914228665, + "tokens_seen": 1671233536 + }, + { + "epoch": 0.01, + "learning_rate": 0.000296, + "loss": 1.5873, + "theoretical_loss": 3.4830597973386683, + "tokens_seen": 1671299072 + }, + { + "epoch": 0.01, + "learning_rate": 0.000298, + "loss": 1.5036, + "theoretical_loss": 3.4830480038464255, + "tokens_seen": 1671364608 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.3960166275501251, + "objective/train/avg_token_score": 0.020477375015616417, + "objective/train/docs_used": 946982, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2867282629013062, + "objective/train/original_loss": 4.248522758483887, + "objective/train/theoretical_loss": 3.4830362109460857, + "objective/train/tokens_used": 30289376, + "objective/train/unlikelihood_loss": 0.06127317622303963, + "theoretical_loss": 3.4830362109460857, + "tokens_seen": 1671430144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003, + "loss": 1.2874, + "theoretical_loss": 3.4830362109460857, + "tokens_seen": 1671430144 + }, + { + "epoch": 0.01, + "learning_rate": 0.000302, + "loss": 1.5552, + "theoretical_loss": 3.483024418637595, + "tokens_seen": 1671495680 + }, + { + "epoch": 0.01, + "learning_rate": 0.000304, + "loss": 1.4532, + "theoretical_loss": 3.4830126269209014, + "tokens_seen": 1671561216 + }, + { + "epoch": 0.01, + "learning_rate": 0.000306, + "loss": 1.3209, + "theoretical_loss": 3.4830008357959517, + "tokens_seen": 1671626752 + }, + { + "epoch": 0.01, + "learning_rate": 0.000308, + "loss": 1.5412, + "theoretical_loss": 3.4829890452626926, + "tokens_seen": 1671692288 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031, + "loss": 1.0823, + "theoretical_loss": 3.482977255321072, + "tokens_seen": 1671757824 + }, + { + "epoch": 0.01, + "learning_rate": 0.000312, + "loss": 1.4433, + "theoretical_loss": 3.482965465971036, + "tokens_seen": 1671823360 + }, + { + "epoch": 0.01, + "learning_rate": 0.000314, + "loss": 1.3343, + "theoretical_loss": 3.4829536772125325, + "tokens_seen": 1671888896 + }, + { + "epoch": 0.01, + "learning_rate": 0.000316, + "loss": 1.4329, + "theoretical_loss": 3.482941889045509, + "tokens_seen": 1671954432 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031800000000000003, + "loss": 1.2974, + "theoretical_loss": 3.4829301014699117, + "tokens_seen": 1672019968 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032, + "loss": 1.5543, + "theoretical_loss": 3.482918314485688, + "tokens_seen": 1672085504 + }, + { + "epoch": 0.01, + "learning_rate": 0.000322, + "loss": 1.6948, + "theoretical_loss": 3.4829065280927853, + "tokens_seen": 1672151040 + }, + { + "epoch": 0.01, + "learning_rate": 0.000324, + "loss": 1.358, + "theoretical_loss": 3.482894742291151, + "tokens_seen": 1672216576 + }, + { + "epoch": 0.01, + "learning_rate": 0.000326, + "loss": 1.5256, + "theoretical_loss": 3.4828829570807316, + "tokens_seen": 1672282112 + }, + { + "epoch": 0.01, + "learning_rate": 0.000328, + "loss": 1.2557, + "theoretical_loss": 3.4828711724614747, + "tokens_seen": 1672347648 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033, + "loss": 1.4626, + "theoretical_loss": 3.4828593884333277, + "tokens_seen": 1672413184 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033200000000000005, + "loss": 1.4653, + "theoretical_loss": 3.4828476049962367, + "tokens_seen": 1672478720 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033400000000000004, + "loss": 1.4492, + "theoretical_loss": 3.4828358221501503, + "tokens_seen": 1672544256 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033600000000000004, + "loss": 1.3397, + "theoretical_loss": 3.4828240398950143, + "tokens_seen": 1672609792 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033800000000000003, + "loss": 1.2303, + "theoretical_loss": 3.4828122582307777, + "tokens_seen": 1672675328 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034, + "loss": 1.3492, + "theoretical_loss": 3.482800477157386, + "tokens_seen": 1672740864 + }, + { + "epoch": 0.01, + "learning_rate": 0.000342, + "loss": 1.4296, + "theoretical_loss": 3.4827886966747874, + "tokens_seen": 1672806400 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034399999999999996, + "loss": 1.6639, + "theoretical_loss": 3.482776916782929, + "tokens_seen": 1672871936 + }, + { + "epoch": 0.01, + "learning_rate": 0.000346, + "loss": 1.3111, + "theoretical_loss": 3.482765137481757, + "tokens_seen": 1672937472 + }, + { + "epoch": 0.01, + "learning_rate": 0.000348, + "loss": 1.4424, + "theoretical_loss": 3.4827533587712205, + "tokens_seen": 1673003008 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.42277616262435913, + "objective/train/avg_token_score": 0.034132879227399826, + "objective/train/docs_used": 947668, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4029995203018188, + "objective/train/original_loss": 4.207383155822754, + "objective/train/theoretical_loss": 3.4827415806512656, + "objective/train/tokens_used": 31927776, + "objective/train/unlikelihood_loss": 0.05666952580213547, + "theoretical_loss": 3.4827415806512656, + "tokens_seen": 1673068544 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035, + "loss": 1.7249, + "theoretical_loss": 3.4827415806512656, + "tokens_seen": 1673068544 + }, + { + "epoch": 0.01, + "learning_rate": 0.000352, + "loss": 1.4411, + "theoretical_loss": 3.4827298031218397, + "tokens_seen": 1673134080 + }, + { + "epoch": 0.01, + "learning_rate": 0.000354, + "loss": 1.4599, + "theoretical_loss": 3.48271802618289, + "tokens_seen": 1673199616 + }, + { + "epoch": 0.01, + "learning_rate": 0.000356, + "loss": 1.4334, + "theoretical_loss": 3.4827062498343637, + "tokens_seen": 1673265152 + }, + { + "epoch": 0.01, + "learning_rate": 0.000358, + "loss": 1.4262, + "theoretical_loss": 3.4826944740762085, + "tokens_seen": 1673330688 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035999999999999997, + "loss": 1.5422, + "theoretical_loss": 3.4826826989083712, + "tokens_seen": 1673396224 + }, + { + "epoch": 0.01, + "learning_rate": 0.000362, + "loss": 1.3631, + "theoretical_loss": 3.4826709243307996, + "tokens_seen": 1673461760 + }, + { + "epoch": 0.01, + "learning_rate": 0.000364, + "loss": 1.5299, + "theoretical_loss": 3.4826591503434408, + "tokens_seen": 1673527296 + }, + { + "epoch": 0.01, + "learning_rate": 0.000366, + "loss": 1.4358, + "theoretical_loss": 3.4826473769462414, + "tokens_seen": 1673592832 + }, + { + "epoch": 0.01, + "learning_rate": 0.000368, + "loss": 1.3031, + "theoretical_loss": 3.4826356041391495, + "tokens_seen": 1673658368 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037, + "loss": 1.372, + "theoretical_loss": 3.482623831922113, + "tokens_seen": 1673723904 + }, + { + "epoch": 0.01, + "learning_rate": 0.000372, + "loss": 1.4469, + "theoretical_loss": 3.4826120602950774, + "tokens_seen": 1673789440 + }, + { + "epoch": 0.01, + "learning_rate": 0.000374, + "loss": 1.3485, + "theoretical_loss": 3.4826002892579915, + "tokens_seen": 1673854976 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037600000000000003, + "loss": 1.1995, + "theoretical_loss": 3.482588518810802, + "tokens_seen": 1673920512 + }, + { + "epoch": 0.01, + "learning_rate": 0.000378, + "loss": 1.4059, + "theoretical_loss": 3.4825767489534574, + "tokens_seen": 1673986048 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038, + "loss": 1.4645, + "theoretical_loss": 3.482564979685903, + "tokens_seen": 1674051584 + }, + { + "epoch": 0.01, + "learning_rate": 0.000382, + "loss": 1.6177, + "theoretical_loss": 3.482553211008088, + "tokens_seen": 1674117120 + }, + { + "epoch": 0.01, + "learning_rate": 0.000384, + "loss": 1.5993, + "theoretical_loss": 3.482541442919959, + "tokens_seen": 1674182656 + }, + { + "epoch": 0.01, + "learning_rate": 0.000386, + "loss": 1.5792, + "theoretical_loss": 3.482529675421463, + "tokens_seen": 1674248192 + }, + { + "epoch": 0.01, + "learning_rate": 0.000388, + "loss": 1.2894, + "theoretical_loss": 3.4825179085125484, + "tokens_seen": 1674313728 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039000000000000005, + "loss": 1.3151, + "theoretical_loss": 3.4825061421931616, + "tokens_seen": 1674379264 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039200000000000004, + "loss": 1.484, + "theoretical_loss": 3.4824943764632503, + "tokens_seen": 1674444800 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039400000000000004, + "loss": 1.4304, + "theoretical_loss": 3.4824826113227623, + "tokens_seen": 1674510336 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039600000000000003, + "loss": 1.3797, + "theoretical_loss": 3.4824708467716445, + "tokens_seen": 1674575872 + }, + { + "epoch": 0.01, + "learning_rate": 0.000398, + "loss": 1.1721, + "theoretical_loss": 3.4824590828098447, + "tokens_seen": 1674641408 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.47403469681739807, + "objective/train/avg_token_score": 0.020986413583159447, + "objective/train/docs_used": 948408, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4957624673843384, + "objective/train/original_loss": 4.087368488311768, + "objective/train/theoretical_loss": 3.48244731943731, + "objective/train/tokens_used": 33566176, + "objective/train/unlikelihood_loss": 0.058626897633075714, + "theoretical_loss": 3.48244731943731, + "tokens_seen": 1674706944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004, + "loss": 1.6842, + "theoretical_loss": 3.48244731943731, + "tokens_seen": 1674706944 + }, + { + "epoch": 0.01, + "learning_rate": 0.000402, + "loss": 1.3746, + "theoretical_loss": 3.4824355566539875, + "tokens_seen": 1674772480 + }, + { + "epoch": 0.01, + "learning_rate": 0.000404, + "loss": 1.4025, + "theoretical_loss": 3.4824237944598257, + "tokens_seen": 1674838016 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040600000000000006, + "loss": 1.4322, + "theoretical_loss": 3.482412032854771, + "tokens_seen": 1674903552 + }, + { + "epoch": 0.01, + "learning_rate": 0.000408, + "loss": 1.5092, + "theoretical_loss": 3.482400271838771, + "tokens_seen": 1674969088 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041, + "loss": 1.3048, + "theoretical_loss": 3.482388511411774, + "tokens_seen": 1675034624 + }, + { + "epoch": 0.01, + "learning_rate": 0.000412, + "loss": 1.1327, + "theoretical_loss": 3.4823767515737267, + "tokens_seen": 1675100160 + }, + { + "epoch": 0.01, + "learning_rate": 0.000414, + "loss": 1.3768, + "theoretical_loss": 3.4823649923245767, + "tokens_seen": 1675165696 + }, + { + "epoch": 0.01, + "learning_rate": 0.000416, + "loss": 1.3944, + "theoretical_loss": 3.4823532336642717, + "tokens_seen": 1675231232 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041799999999999997, + "loss": 1.5612, + "theoretical_loss": 3.482341475592759, + "tokens_seen": 1675296768 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042, + "loss": 1.6033, + "theoretical_loss": 3.4823297181099857, + "tokens_seen": 1675362304 + }, + { + "epoch": 0.01, + "learning_rate": 0.000422, + "loss": 1.2907, + "theoretical_loss": 3.4823179612159, + "tokens_seen": 1675427840 + }, + { + "epoch": 0.01, + "learning_rate": 0.000424, + "loss": 1.4496, + "theoretical_loss": 3.4823062049104485, + "tokens_seen": 1675493376 + }, + { + "epoch": 0.01, + "learning_rate": 0.000426, + "loss": 1.3219, + "theoretical_loss": 3.4822944491935797, + "tokens_seen": 1675558912 + }, + { + "epoch": 0.01, + "learning_rate": 0.000428, + "loss": 1.6165, + "theoretical_loss": 3.4822826940652405, + "tokens_seen": 1675624448 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043, + "loss": 1.46, + "theoretical_loss": 3.482270939525379, + "tokens_seen": 1675689984 + }, + { + "epoch": 0.01, + "learning_rate": 0.000432, + "loss": 1.3409, + "theoretical_loss": 3.482259185573942, + "tokens_seen": 1675755520 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043400000000000003, + "loss": 1.5918, + "theoretical_loss": 3.4822474322108774, + "tokens_seen": 1675821056 + }, + { + "epoch": 0.01, + "learning_rate": 0.000436, + "loss": 1.3411, + "theoretical_loss": 3.4822356794361324, + "tokens_seen": 1675886592 + }, + { + "epoch": 0.01, + "learning_rate": 0.000438, + "loss": 1.4671, + "theoretical_loss": 3.482223927249655, + "tokens_seen": 1675952128 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044, + "loss": 1.407, + "theoretical_loss": 3.482212175651393, + "tokens_seen": 1676017664 + }, + { + "epoch": 0.01, + "learning_rate": 0.000442, + "loss": 1.4158, + "theoretical_loss": 3.4822004246412934, + "tokens_seen": 1676083200 + }, + { + "epoch": 0.01, + "learning_rate": 0.000444, + "loss": 1.2442, + "theoretical_loss": 3.4821886742193042, + "tokens_seen": 1676148736 + }, + { + "epoch": 0.01, + "learning_rate": 0.000446, + "loss": 1.6039, + "theoretical_loss": 3.4821769243853717, + "tokens_seen": 1676214272 + }, + { + "epoch": 0.01, + "learning_rate": 0.000448, + "loss": 1.3757, + "theoretical_loss": 3.4821651751394453, + "tokens_seen": 1676279808 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.43438416719436646, + "objective/train/avg_token_score": 0.02250559814274311, + "objective/train/docs_used": 949481, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3553886413574219, + "objective/train/original_loss": 4.167667388916016, + "objective/train/theoretical_loss": 3.4821534264814717, + "objective/train/tokens_used": 35204576, + "objective/train/unlikelihood_loss": 0.05399244651198387, + "theoretical_loss": 3.4821534264814717, + "tokens_seen": 1676345344 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045000000000000004, + "loss": 1.5774, + "theoretical_loss": 3.4821534264814717, + "tokens_seen": 1676345344 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045200000000000004, + "loss": 1.1835, + "theoretical_loss": 3.4821416784113985, + "tokens_seen": 1676410880 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045400000000000003, + "loss": 1.4729, + "theoretical_loss": 3.4821299309291733, + "tokens_seen": 1676476416 + }, + { + "epoch": 0.01, + "learning_rate": 0.000456, + "loss": 1.4368, + "theoretical_loss": 3.4821181840347437, + "tokens_seen": 1676541952 + }, + { + "epoch": 0.01, + "learning_rate": 0.000458, + "loss": 1.5042, + "theoretical_loss": 3.4821064377280577, + "tokens_seen": 1676607488 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046, + "loss": 1.4255, + "theoretical_loss": 3.4820946920090625, + "tokens_seen": 1676673024 + }, + { + "epoch": 0.01, + "learning_rate": 0.000462, + "loss": 1.3812, + "theoretical_loss": 3.4820829468777057, + "tokens_seen": 1676738560 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046400000000000006, + "loss": 1.384, + "theoretical_loss": 3.482071202333935, + "tokens_seen": 1676804096 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046600000000000005, + "loss": 1.3564, + "theoretical_loss": 3.482059458377698, + "tokens_seen": 1676869632 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046800000000000005, + "loss": 1.4955, + "theoretical_loss": 3.4820477150089433, + "tokens_seen": 1676935168 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047, + "loss": 1.3263, + "theoretical_loss": 3.4820359722276164, + "tokens_seen": 1677000704 + }, + { + "epoch": 0.01, + "learning_rate": 0.000472, + "loss": 1.6779, + "theoretical_loss": 3.4820242300336677, + "tokens_seen": 1677066240 + }, + { + "epoch": 0.01, + "learning_rate": 0.000474, + "loss": 1.4524, + "theoretical_loss": 3.4820124884270425, + "tokens_seen": 1677131776 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047599999999999997, + "loss": 1.5956, + "theoretical_loss": 3.4820007474076893, + "tokens_seen": 1677197312 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047799999999999996, + "loss": 1.5913, + "theoretical_loss": 3.481989006975556, + "tokens_seen": 1677262848 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048, + "loss": 1.4133, + "theoretical_loss": 3.4819772671305906, + "tokens_seen": 1677328384 + }, + { + "epoch": 0.01, + "learning_rate": 0.000482, + "loss": 1.2181, + "theoretical_loss": 3.48196552787274, + "tokens_seen": 1677393920 + }, + { + "epoch": 0.01, + "learning_rate": 0.000484, + "loss": 1.337, + "theoretical_loss": 3.4819537892019525, + "tokens_seen": 1677459456 + }, + { + "epoch": 0.01, + "learning_rate": 0.000486, + "loss": 1.2865, + "theoretical_loss": 3.4819420511181756, + "tokens_seen": 1677524992 + }, + { + "epoch": 0.01, + "learning_rate": 0.000488, + "loss": 1.6925, + "theoretical_loss": 3.481930313621357, + "tokens_seen": 1677590528 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049, + "loss": 1.2952, + "theoretical_loss": 3.4819185767114442, + "tokens_seen": 1677656064 + }, + { + "epoch": 0.01, + "learning_rate": 0.000492, + "loss": 1.3687, + "theoretical_loss": 3.4819068403883855, + "tokens_seen": 1677721600 + }, + { + "epoch": 0.01, + "learning_rate": 0.000494, + "loss": 1.4566, + "theoretical_loss": 3.4818951046521276, + "tokens_seen": 1677787136 + }, + { + "epoch": 0.01, + "learning_rate": 0.000496, + "loss": 1.3783, + "theoretical_loss": 3.4818833695026195, + "tokens_seen": 1677852672 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498, + "loss": 1.4967, + "theoretical_loss": 3.4818716349398082, + "tokens_seen": 1677918208 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.42913001775741577, + "objective/train/avg_token_score": 0.028014685958623886, + "objective/train/docs_used": 950694, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2823351621627808, + "objective/train/original_loss": 4.018346309661865, + "objective/train/theoretical_loss": 3.481859900963642, + "objective/train/tokens_used": 36842976, + "objective/train/unlikelihood_loss": 0.06534267216920853, + "theoretical_loss": 3.481859900963642, + "tokens_seen": 1677983744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005, + "loss": 1.4589, + "theoretical_loss": 3.481859900963642, + "tokens_seen": 1677983744 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499979797979798, + "loss": 1.7066, + "theoretical_loss": 3.481848167574068, + "tokens_seen": 1678049280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999595959595959, + "loss": 1.4743, + "theoretical_loss": 3.481836434771034, + "tokens_seen": 1678114816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999393939393939, + "loss": 1.4814, + "theoretical_loss": 3.4818247025544884, + "tokens_seen": 1678180352 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499919191919192, + "loss": 1.5105, + "theoretical_loss": 3.481812970924379, + "tokens_seen": 1678245888 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049989898989899, + "loss": 1.3163, + "theoretical_loss": 3.481801239880652, + "tokens_seen": 1678311424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998787878787879, + "loss": 1.5049, + "theoretical_loss": 3.4817895094232574, + "tokens_seen": 1678376960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998585858585859, + "loss": 1.7115, + "theoretical_loss": 3.481777779552141, + "tokens_seen": 1678442496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998383838383839, + "loss": 1.3624, + "theoretical_loss": 3.481766050267253, + "tokens_seen": 1678508032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998181818181818, + "loss": 1.4109, + "theoretical_loss": 3.481754321568539, + "tokens_seen": 1678573568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997979797979798, + "loss": 1.6151, + "theoretical_loss": 3.4817425934559476, + "tokens_seen": 1678639104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997777777777778, + "loss": 1.4839, + "theoretical_loss": 3.4817308659294266, + "tokens_seen": 1678704640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997575757575757, + "loss": 1.3913, + "theoretical_loss": 3.4817191389889244, + "tokens_seen": 1678770176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997373737373737, + "loss": 1.4744, + "theoretical_loss": 3.481707412634388, + "tokens_seen": 1678835712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997171717171717, + "loss": 1.2652, + "theoretical_loss": 3.481695686865766, + "tokens_seen": 1678901248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996969696969697, + "loss": 1.5878, + "theoretical_loss": 3.481683961683005, + "tokens_seen": 1678966784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996767676767676, + "loss": 1.2625, + "theoretical_loss": 3.4816722370860544, + "tokens_seen": 1679032320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996565656565657, + "loss": 1.3929, + "theoretical_loss": 3.4816605130748615, + "tokens_seen": 1679097856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996363636363637, + "loss": 1.4169, + "theoretical_loss": 3.4816487896493733, + "tokens_seen": 1679163392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996161616161617, + "loss": 1.4403, + "theoretical_loss": 3.4816370668095393, + "tokens_seen": 1679228928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995959595959596, + "loss": 1.5393, + "theoretical_loss": 3.4816253445553063, + "tokens_seen": 1679294464 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995757575757576, + "loss": 1.6212, + "theoretical_loss": 3.4816136228866217, + "tokens_seen": 1679360000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995555555555555, + "loss": 1.4312, + "theoretical_loss": 3.481601901803435, + "tokens_seen": 1679425536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995353535353536, + "loss": 1.4936, + "theoretical_loss": 3.481590181305693, + "tokens_seen": 1679491072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995151515151515, + "loss": 1.29, + "theoretical_loss": 3.4815784613933434, + "tokens_seen": 1679556608 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.3876466155052185, + "objective/train/avg_token_score": 0.019771317020058632, + "objective/train/docs_used": 951339, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2079968452453613, + "objective/train/original_loss": 3.654224395751953, + "objective/train/theoretical_loss": 3.481566742066335, + "objective/train/tokens_used": 38481376, + "objective/train/unlikelihood_loss": 0.10271399468183517, + "theoretical_loss": 3.481566742066335, + "tokens_seen": 1679622144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994949494949495, + "loss": 1.1997, + "theoretical_loss": 3.481566742066335, + "tokens_seen": 1679622144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994747474747474, + "loss": 1.3423, + "theoretical_loss": 3.4815550233246153, + "tokens_seen": 1679687680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994545454545454, + "loss": 1.416, + "theoretical_loss": 3.481543305168132, + "tokens_seen": 1679753216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994343434343435, + "loss": 1.4918, + "theoretical_loss": 3.4815315875968333, + "tokens_seen": 1679818752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994141414141415, + "loss": 1.6278, + "theoretical_loss": 3.4815198706106676, + "tokens_seen": 1679884288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993939393939394, + "loss": 1.4463, + "theoretical_loss": 3.481508154209582, + "tokens_seen": 1679949824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993737373737374, + "loss": 1.4316, + "theoretical_loss": 3.4814964383935245, + "tokens_seen": 1680015360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993535353535354, + "loss": 1.5408, + "theoretical_loss": 3.4814847231624437, + "tokens_seen": 1680080896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993333333333334, + "loss": 1.4201, + "theoretical_loss": 3.4814730085162875, + "tokens_seen": 1680146432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993131313131313, + "loss": 1.3325, + "theoretical_loss": 3.4814612944550034, + "tokens_seen": 1680211968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992929292929293, + "loss": 1.4038, + "theoretical_loss": 3.4814495809785395, + "tokens_seen": 1680277504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992727272727272, + "loss": 1.4769, + "theoretical_loss": 3.4814378680868447, + "tokens_seen": 1680343040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992525252525252, + "loss": 1.5646, + "theoretical_loss": 3.4814261557798654, + "tokens_seen": 1680408576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992323232323232, + "loss": 1.3709, + "theoretical_loss": 3.4814144440575507, + "tokens_seen": 1680474112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992121212121212, + "loss": 1.3969, + "theoretical_loss": 3.4814027329198485, + "tokens_seen": 1680539648 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991919191919192, + "loss": 1.4675, + "theoretical_loss": 3.481391022366707, + "tokens_seen": 1680605184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991717171717172, + "loss": 1.5211, + "theoretical_loss": 3.481379312398073, + "tokens_seen": 1680670720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991515151515152, + "loss": 1.424, + "theoretical_loss": 3.4813676030138963, + "tokens_seen": 1680736256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991313131313132, + "loss": 1.285, + "theoretical_loss": 3.481355894214124, + "tokens_seen": 1680801792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991111111111111, + "loss": 1.5255, + "theoretical_loss": 3.481344185998704, + "tokens_seen": 1680867328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990909090909091, + "loss": 1.3467, + "theoretical_loss": 3.4813324783675847, + "tokens_seen": 1680932864 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990707070707071, + "loss": 1.414, + "theoretical_loss": 3.481320771320714, + "tokens_seen": 1680998400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990505050505051, + "loss": 1.4323, + "theoretical_loss": 3.4813090648580403, + "tokens_seen": 1681063936 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499030303030303, + "loss": 1.2216, + "theoretical_loss": 3.4812973589795106, + "tokens_seen": 1681129472 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499010101010101, + "loss": 1.6606, + "theoretical_loss": 3.4812856536850747, + "tokens_seen": 1681195008 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.57746821641922, + "objective/train/avg_token_score": 0.003087435383349657, + "objective/train/docs_used": 952727, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.965943694114685, + "objective/train/original_loss": 3.664557933807373, + "objective/train/theoretical_loss": 3.4812739489746796, + "objective/train/tokens_used": 40119776, + "objective/train/unlikelihood_loss": 0.07379762828350067, + "theoretical_loss": 3.4812739489746796, + "tokens_seen": 1681260544 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498989898989899, + "loss": 1.3937, + "theoretical_loss": 3.4812739489746796, + "tokens_seen": 1681260544 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498969696969697, + "loss": 1.4396, + "theoretical_loss": 3.4812622448482733, + "tokens_seen": 1681326080 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498949494949495, + "loss": 1.5074, + "theoretical_loss": 3.481250541305804, + "tokens_seen": 1681391616 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498929292929293, + "loss": 1.3075, + "theoretical_loss": 3.4812388383472204, + "tokens_seen": 1681457152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989090909090909, + "loss": 1.4787, + "theoretical_loss": 3.4812271359724702, + "tokens_seen": 1681522688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988888888888889, + "loss": 1.4177, + "theoretical_loss": 3.481215434181501, + "tokens_seen": 1681588224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988686868686869, + "loss": 1.3858, + "theoretical_loss": 3.481203732974262, + "tokens_seen": 1681653760 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988484848484849, + "loss": 1.5262, + "theoretical_loss": 3.481192032350701, + "tokens_seen": 1681719296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988282828282828, + "loss": 1.5809, + "theoretical_loss": 3.481180332310765, + "tokens_seen": 1681784832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988080808080808, + "loss": 1.505, + "theoretical_loss": 3.481168632854404, + "tokens_seen": 1681850368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987878787878788, + "loss": 1.3779, + "theoretical_loss": 3.4811569339815644, + "tokens_seen": 1681915904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987676767676767, + "loss": 1.3716, + "theoretical_loss": 3.4811452356921953, + "tokens_seen": 1681981440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987474747474747, + "loss": 1.572, + "theoretical_loss": 3.4811335379862456, + "tokens_seen": 1682046976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987272727272728, + "loss": 1.4112, + "theoretical_loss": 3.4811218408636617, + "tokens_seen": 1682112512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987070707070708, + "loss": 1.4665, + "theoretical_loss": 3.481110144324393, + "tokens_seen": 1682178048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986868686868687, + "loss": 1.4609, + "theoretical_loss": 3.481098448368387, + "tokens_seen": 1682243584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986666666666667, + "loss": 1.6894, + "theoretical_loss": 3.4810867529955933, + "tokens_seen": 1682309120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986464646464647, + "loss": 1.3648, + "theoretical_loss": 3.481075058205958, + "tokens_seen": 1682374656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986262626262627, + "loss": 1.5816, + "theoretical_loss": 3.4810633639994313, + "tokens_seen": 1682440192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986060606060606, + "loss": 1.6544, + "theoretical_loss": 3.48105167037596, + "tokens_seen": 1682505728 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985858585858586, + "loss": 1.4945, + "theoretical_loss": 3.4810399773354925, + "tokens_seen": 1682571264 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985656565656565, + "loss": 1.3454, + "theoretical_loss": 3.4810282848779783, + "tokens_seen": 1682636800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985454545454545, + "loss": 1.2652, + "theoretical_loss": 3.481016593003364, + "tokens_seen": 1682702336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985252525252525, + "loss": 1.3969, + "theoretical_loss": 3.4810049017115983, + "tokens_seen": 1682767872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985050505050506, + "loss": 1.5008, + "theoretical_loss": 3.4809932110026303, + "tokens_seen": 1682833408 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.4591275751590729, + "objective/train/avg_token_score": 0.02467002533376217, + "objective/train/docs_used": 953335, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5312504768371582, + "objective/train/original_loss": 4.14010763168335, + "objective/train/theoretical_loss": 3.4809815208764077, + "objective/train/tokens_used": 41758176, + "objective/train/unlikelihood_loss": 0.059290263801813126, + "theoretical_loss": 3.4809815208764077, + "tokens_seen": 1682898944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984848484848485, + "loss": 1.423, + "theoretical_loss": 3.4809815208764077, + "tokens_seen": 1682898944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984646464646465, + "loss": 1.6025, + "theoretical_loss": 3.480969831332878, + "tokens_seen": 1682964480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984444444444445, + "loss": 1.6665, + "theoretical_loss": 3.4809581423719904, + "tokens_seen": 1683030016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984242424242425, + "loss": 1.472, + "theoretical_loss": 3.480946453993693, + "tokens_seen": 1683095552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984040404040404, + "loss": 1.4715, + "theoretical_loss": 3.480934766197934, + "tokens_seen": 1683161088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983838383838384, + "loss": 1.4211, + "theoretical_loss": 3.480923078984662, + "tokens_seen": 1683226624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983636363636363, + "loss": 1.495, + "theoretical_loss": 3.480911392353825, + "tokens_seen": 1683292160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983434343434344, + "loss": 1.2509, + "theoretical_loss": 3.4808997063053706, + "tokens_seen": 1683357696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983232323232323, + "loss": 1.3817, + "theoretical_loss": 3.480888020839248, + "tokens_seen": 1683423232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983030303030303, + "loss": 1.2817, + "theoretical_loss": 3.480876335955406, + "tokens_seen": 1683488768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982828282828282, + "loss": 1.4437, + "theoretical_loss": 3.4808646516537918, + "tokens_seen": 1683554304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982626262626263, + "loss": 1.5573, + "theoretical_loss": 3.4808529679343536, + "tokens_seen": 1683619840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982424242424243, + "loss": 1.6952, + "theoretical_loss": 3.480841284797041, + "tokens_seen": 1683685376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982222222222223, + "loss": 1.2028, + "theoretical_loss": 3.4808296022418013, + "tokens_seen": 1683750912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982020202020202, + "loss": 1.3336, + "theoretical_loss": 3.480817920268583, + "tokens_seen": 1683816448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981818181818182, + "loss": 1.3224, + "theoretical_loss": 3.480806238877335, + "tokens_seen": 1683881984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981616161616162, + "loss": 1.392, + "theoretical_loss": 3.4807945580680055, + "tokens_seen": 1683947520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981414141414142, + "loss": 1.3183, + "theoretical_loss": 3.480782877840542, + "tokens_seen": 1684013056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981212121212121, + "loss": 1.5275, + "theoretical_loss": 3.480771198194894, + "tokens_seen": 1684078592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981010101010101, + "loss": 1.3843, + "theoretical_loss": 3.4807595191310092, + "tokens_seen": 1684144128 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498080808080808, + "loss": 1.4248, + "theoretical_loss": 3.4807478406488364, + "tokens_seen": 1684209664 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498060606060606, + "loss": 1.5212, + "theoretical_loss": 3.4807361627483235, + "tokens_seen": 1684275200 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498040404040404, + "loss": 1.5897, + "theoretical_loss": 3.4807244854294197, + "tokens_seen": 1684340736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980202020202021, + "loss": 1.4325, + "theoretical_loss": 3.480712808692072, + "tokens_seen": 1684406272 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498, + "loss": 1.4759, + "theoretical_loss": 3.4807011325362307, + "tokens_seen": 1684471808 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.46138808131217957, + "objective/train/avg_token_score": 0.026906196027994156, + "objective/train/docs_used": 954676, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2592566013336182, + "objective/train/original_loss": 3.659281015396118, + "objective/train/theoretical_loss": 3.4806894569618425, + "objective/train/tokens_used": 43396576, + "objective/train/unlikelihood_loss": 0.058182328939437866, + "theoretical_loss": 3.4806894569618425, + "tokens_seen": 1684537344 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497979797979798, + "loss": 1.2782, + "theoretical_loss": 3.4806894569618425, + "tokens_seen": 1684537344 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497959595959596, + "loss": 1.3906, + "theoretical_loss": 3.480677781968857, + "tokens_seen": 1684602880 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497939393939394, + "loss": 1.5195, + "theoretical_loss": 3.480666107557221, + "tokens_seen": 1684668416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979191919191919, + "loss": 1.494, + "theoretical_loss": 3.4806544337268854, + "tokens_seen": 1684733952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978989898989899, + "loss": 1.5243, + "theoretical_loss": 3.4806427604777967, + "tokens_seen": 1684799488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978787878787879, + "loss": 1.3817, + "theoretical_loss": 3.4806310878099045, + "tokens_seen": 1684865024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978585858585858, + "loss": 1.3818, + "theoretical_loss": 3.4806194157231567, + "tokens_seen": 1684930560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978383838383838, + "loss": 1.3979, + "theoretical_loss": 3.480607744217502, + "tokens_seen": 1684996096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978181818181818, + "loss": 1.5915, + "theoretical_loss": 3.4805960732928876, + "tokens_seen": 1685061632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977979797979799, + "loss": 1.6397, + "theoretical_loss": 3.480584402949264, + "tokens_seen": 1685127168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977777777777778, + "loss": 1.7836, + "theoretical_loss": 3.4805727331865786, + "tokens_seen": 1685192704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977575757575758, + "loss": 1.4281, + "theoretical_loss": 3.48056106400478, + "tokens_seen": 1685258240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977373737373738, + "loss": 1.5813, + "theoretical_loss": 3.480549395403817, + "tokens_seen": 1685323776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977171717171717, + "loss": 1.5063, + "theoretical_loss": 3.4805377273836378, + "tokens_seen": 1685389312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976969696969697, + "loss": 1.3745, + "theoretical_loss": 3.4805260599441903, + "tokens_seen": 1685454848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976767676767677, + "loss": 1.5523, + "theoretical_loss": 3.480514393085424, + "tokens_seen": 1685520384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976565656565656, + "loss": 1.3951, + "theoretical_loss": 3.4805027268072877, + "tokens_seen": 1685585920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976363636363636, + "loss": 1.3416, + "theoretical_loss": 3.4804910611097286, + "tokens_seen": 1685651456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976161616161616, + "loss": 1.5686, + "theoretical_loss": 3.4804793959926963, + "tokens_seen": 1685716992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975959595959596, + "loss": 1.489, + "theoretical_loss": 3.480467731456139, + "tokens_seen": 1685782528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975757575757575, + "loss": 1.341, + "theoretical_loss": 3.4804560675000054, + "tokens_seen": 1685848064 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975555555555556, + "loss": 1.2872, + "theoretical_loss": 3.4804444041242437, + "tokens_seen": 1685913600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975353535353536, + "loss": 1.5454, + "theoretical_loss": 3.480432741328803, + "tokens_seen": 1685979136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975151515151516, + "loss": 1.5041, + "theoretical_loss": 3.4804210791136314, + "tokens_seen": 1686044672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004974949494949495, + "loss": 1.5567, + "theoretical_loss": 3.4804094174786773, + "tokens_seen": 1686110208 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.5329912304878235, + "objective/train/avg_token_score": 0.01913226768374443, + "objective/train/docs_used": 955358, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6620068550109863, + "objective/train/original_loss": 3.886705160140991, + "objective/train/theoretical_loss": 3.4803977564238897, + "objective/train/tokens_used": 45034976, + "objective/train/unlikelihood_loss": 0.04157567396759987, + "theoretical_loss": 3.4803977564238897, + "tokens_seen": 1686175744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004974747474747475, + "loss": 1.5445, + "theoretical_loss": 3.4803977564238897, + "tokens_seen": 1686175744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974545454545454, + "loss": 1.4179, + "theoretical_loss": 3.4803860959492177, + "tokens_seen": 1686241280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974343434343435, + "loss": 1.4412, + "theoretical_loss": 3.4803744360546087, + "tokens_seen": 1686306816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974141414141414, + "loss": 1.5498, + "theoretical_loss": 3.4803627767400123, + "tokens_seen": 1686372352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973939393939394, + "loss": 1.4011, + "theoretical_loss": 3.480351118005376, + "tokens_seen": 1686437888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973737373737373, + "loss": 1.4917, + "theoretical_loss": 3.48033945985065, + "tokens_seen": 1686503424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973535353535353, + "loss": 1.392, + "theoretical_loss": 3.4803278022757818, + "tokens_seen": 1686568960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973333333333334, + "loss": 1.4304, + "theoretical_loss": 3.4803161452807196, + "tokens_seen": 1686634496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973131313131314, + "loss": 1.4172, + "theoretical_loss": 3.4803044888654133, + "tokens_seen": 1686700032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972929292929293, + "loss": 1.2836, + "theoretical_loss": 3.480292833029811, + "tokens_seen": 1686765568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972727272727273, + "loss": 1.5495, + "theoretical_loss": 3.480281177773861, + "tokens_seen": 1686831104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972525252525253, + "loss": 1.3092, + "theoretical_loss": 3.4802695230975127, + "tokens_seen": 1686896640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972323232323233, + "loss": 1.3053, + "theoretical_loss": 3.480257869000714, + "tokens_seen": 1686962176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972121212121212, + "loss": 1.3317, + "theoretical_loss": 3.4802462154834135, + "tokens_seen": 1687027712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971919191919192, + "loss": 1.484, + "theoretical_loss": 3.4802345625455606, + "tokens_seen": 1687093248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971717171717171, + "loss": 1.4059, + "theoretical_loss": 3.480222910187104, + "tokens_seen": 1687158784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971515151515152, + "loss": 1.5274, + "theoretical_loss": 3.4802112584079916, + "tokens_seen": 1687224320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971313131313131, + "loss": 1.1674, + "theoretical_loss": 3.4801996072081725, + "tokens_seen": 1687289856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971111111111111, + "loss": 1.721, + "theoretical_loss": 3.4801879565875957, + "tokens_seen": 1687355392 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497090909090909, + "loss": 1.2672, + "theoretical_loss": 3.4801763065462095, + "tokens_seen": 1687420928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970707070707071, + "loss": 1.6776, + "theoretical_loss": 3.4801646570839626, + "tokens_seen": 1687486464 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970505050505051, + "loss": 1.3374, + "theoretical_loss": 3.480153008200804, + "tokens_seen": 1687552000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970303030303031, + "loss": 1.2859, + "theoretical_loss": 3.480141359896683, + "tokens_seen": 1687617536 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497010101010101, + "loss": 1.5063, + "theoretical_loss": 3.4801297121715464, + "tokens_seen": 1687683072 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496989898989899, + "loss": 1.3917, + "theoretical_loss": 3.4801180650253447, + "tokens_seen": 1687748608 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.4268084168434143, + "objective/train/avg_token_score": 0.010676675476133823, + "objective/train/docs_used": 956671, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4835927486419678, + "objective/train/original_loss": 4.212736129760742, + "objective/train/theoretical_loss": 3.480106418458026, + "objective/train/tokens_used": 46673376, + "objective/train/unlikelihood_loss": 0.08470705896615982, + "theoretical_loss": 3.480106418458026, + "tokens_seen": 1687814144 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496969696969697, + "loss": 1.4357, + "theoretical_loss": 3.480106418458026, + "tokens_seen": 1687814144 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496949494949495, + "loss": 1.3479, + "theoretical_loss": 3.4800947724695392, + "tokens_seen": 1687879680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969292929292929, + "loss": 1.2908, + "theoretical_loss": 3.4800831270598334, + "tokens_seen": 1687945216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969090909090909, + "loss": 1.506, + "theoretical_loss": 3.4800714822288565, + "tokens_seen": 1688010752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968888888888889, + "loss": 1.5604, + "theoretical_loss": 3.4800598379765577, + "tokens_seen": 1688076288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968686868686868, + "loss": 1.435, + "theoretical_loss": 3.4800481943028863, + "tokens_seen": 1688141824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968484848484849, + "loss": 1.3855, + "theoretical_loss": 3.48003655120779, + "tokens_seen": 1688207360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968282828282829, + "loss": 1.579, + "theoretical_loss": 3.4800249086912185, + "tokens_seen": 1688272896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968080808080808, + "loss": 1.5069, + "theoretical_loss": 3.4800132667531205, + "tokens_seen": 1688338432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967878787878788, + "loss": 1.5546, + "theoretical_loss": 3.4800016253934447, + "tokens_seen": 1688403968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967676767676768, + "loss": 1.2675, + "theoretical_loss": 3.4799899846121396, + "tokens_seen": 1688469504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967474747474748, + "loss": 1.5451, + "theoretical_loss": 3.4799783444091545, + "tokens_seen": 1688535040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967272727272727, + "loss": 1.4007, + "theoretical_loss": 3.4799667047844376, + "tokens_seen": 1688600576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967070707070707, + "loss": 1.4922, + "theoretical_loss": 3.479955065737938, + "tokens_seen": 1688666112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966868686868687, + "loss": 1.6596, + "theoretical_loss": 3.479943427269605, + "tokens_seen": 1688731648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966666666666666, + "loss": 1.7516, + "theoretical_loss": 3.4799317893793873, + "tokens_seen": 1688797184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966464646464646, + "loss": 1.437, + "theoretical_loss": 3.479920152067233, + "tokens_seen": 1688862720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966262626262627, + "loss": 1.3793, + "theoretical_loss": 3.4799085153330918, + "tokens_seen": 1688928256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966060606060607, + "loss": 1.6005, + "theoretical_loss": 3.479896879176912, + "tokens_seen": 1688993792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965858585858586, + "loss": 1.5023, + "theoretical_loss": 3.479885243598643, + "tokens_seen": 1689059328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965656565656566, + "loss": 1.4687, + "theoretical_loss": 3.4798736085982336, + "tokens_seen": 1689124864 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965454545454546, + "loss": 1.6716, + "theoretical_loss": 3.479861974175632, + "tokens_seen": 1689190400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965252525252525, + "loss": 1.6124, + "theoretical_loss": 3.4798503403307874, + "tokens_seen": 1689255936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965050505050505, + "loss": 1.2172, + "theoretical_loss": 3.4798387070636494, + "tokens_seen": 1689321472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964848484848485, + "loss": 1.7067, + "theoretical_loss": 3.479827074374166, + "tokens_seen": 1689387008 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.4199046790599823, + "objective/train/avg_token_score": 0.011589493602514267, + "objective/train/docs_used": 957370, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3580185174942017, + "objective/train/original_loss": 4.241112232208252, + "objective/train/theoretical_loss": 3.4798154422622867, + "objective/train/tokens_used": 48311776, + "objective/train/unlikelihood_loss": 0.06464245170354843, + "theoretical_loss": 3.4798154422622867, + "tokens_seen": 1689452544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964646464646464, + "loss": 1.4656, + "theoretical_loss": 3.4798154422622867, + "tokens_seen": 1689452544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964444444444444, + "loss": 1.4522, + "theoretical_loss": 3.4798038107279603, + "tokens_seen": 1689518080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964242424242424, + "loss": 1.5003, + "theoretical_loss": 3.4797921797711355, + "tokens_seen": 1689583616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964040404040405, + "loss": 1.4859, + "theoretical_loss": 3.4797805493917613, + "tokens_seen": 1689649152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963838383838384, + "loss": 1.5211, + "theoretical_loss": 3.4797689195897865, + "tokens_seen": 1689714688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963636363636364, + "loss": 1.5367, + "theoretical_loss": 3.4797572903651606, + "tokens_seen": 1689780224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963434343434344, + "loss": 1.4631, + "theoretical_loss": 3.479745661717832, + "tokens_seen": 1689845760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963232323232324, + "loss": 1.4319, + "theoretical_loss": 3.47973403364775, + "tokens_seen": 1689911296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963030303030303, + "loss": 1.2408, + "theoretical_loss": 3.4797224061548633, + "tokens_seen": 1689976832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962828282828283, + "loss": 1.3776, + "theoretical_loss": 3.4797107792391206, + "tokens_seen": 1690042368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962626262626262, + "loss": 1.4295, + "theoretical_loss": 3.4796991529004715, + "tokens_seen": 1690107904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962424242424243, + "loss": 1.5602, + "theoretical_loss": 3.4796875271388648, + "tokens_seen": 1690173440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962222222222222, + "loss": 1.3299, + "theoretical_loss": 3.4796759019542494, + "tokens_seen": 1690238976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962020202020202, + "loss": 1.4424, + "theoretical_loss": 3.479664277346574, + "tokens_seen": 1690304512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961818181818181, + "loss": 1.6119, + "theoretical_loss": 3.479652653315788, + "tokens_seen": 1690370048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961616161616161, + "loss": 1.5685, + "theoretical_loss": 3.4796410298618405, + "tokens_seen": 1690435584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961414141414142, + "loss": 1.4242, + "theoretical_loss": 3.4796294069846803, + "tokens_seen": 1690501120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961212121212122, + "loss": 1.4835, + "theoretical_loss": 3.4796177846842564, + "tokens_seen": 1690566656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961010101010101, + "loss": 1.3436, + "theoretical_loss": 3.479606162960518, + "tokens_seen": 1690632192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960808080808081, + "loss": 1.6164, + "theoretical_loss": 3.4795945418134133, + "tokens_seen": 1690697728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960606060606061, + "loss": 1.6758, + "theoretical_loss": 3.479582921242893, + "tokens_seen": 1690763264 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960404040404041, + "loss": 1.298, + "theoretical_loss": 3.4795713012489045, + "tokens_seen": 1690828800 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496020202020202, + "loss": 1.4461, + "theoretical_loss": 3.4795596818313976, + "tokens_seen": 1690894336 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496, + "loss": 1.4362, + "theoretical_loss": 3.4795480629903217, + "tokens_seen": 1690959872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959797979797979, + "loss": 1.3436, + "theoretical_loss": 3.479536444725625, + "tokens_seen": 1691025408 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.38599705696105957, + "objective/train/avg_token_score": 0.048682406544685364, + "objective/train/docs_used": 958661, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2436673641204834, + "objective/train/original_loss": 4.153236389160156, + "objective/train/theoretical_loss": 3.4795248270372574, + "objective/train/tokens_used": 49950176, + "objective/train/unlikelihood_loss": 0.07140035182237625, + "theoretical_loss": 3.4795248270372574, + "tokens_seen": 1691090944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959595959595959, + "loss": 1.2475, + "theoretical_loss": 3.4795248270372574, + "tokens_seen": 1691090944 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495939393939394, + "loss": 1.6063, + "theoretical_loss": 3.479513209925167, + "tokens_seen": 1691156480 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495919191919192, + "loss": 1.4383, + "theoretical_loss": 3.479501593389304, + "tokens_seen": 1691222016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958989898989899, + "loss": 1.6435, + "theoretical_loss": 3.479489977429617, + "tokens_seen": 1691287552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958787878787879, + "loss": 1.5405, + "theoretical_loss": 3.4794783620460548, + "tokens_seen": 1691353088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958585858585859, + "loss": 1.5615, + "theoretical_loss": 3.479466747238567, + "tokens_seen": 1691418624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958383838383839, + "loss": 1.51, + "theoretical_loss": 3.4794551330071024, + "tokens_seen": 1691484160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958181818181818, + "loss": 1.4175, + "theoretical_loss": 3.47944351935161, + "tokens_seen": 1691549696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957979797979798, + "loss": 1.5496, + "theoretical_loss": 3.4794319062720396, + "tokens_seen": 1691615232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957777777777778, + "loss": 1.4502, + "theoretical_loss": 3.479420293768339, + "tokens_seen": 1691680768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957575757575757, + "loss": 1.4327, + "theoretical_loss": 3.479408681840459, + "tokens_seen": 1691746304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957373737373737, + "loss": 1.4366, + "theoretical_loss": 3.4793970704883477, + "tokens_seen": 1691811840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957171717171717, + "loss": 1.2897, + "theoretical_loss": 3.4793854597119545, + "tokens_seen": 1691877376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956969696969698, + "loss": 1.6798, + "theoretical_loss": 3.4793738495112283, + "tokens_seen": 1691942912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956767676767677, + "loss": 1.2665, + "theoretical_loss": 3.4793622398861186, + "tokens_seen": 1692008448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956565656565657, + "loss": 1.5467, + "theoretical_loss": 3.4793506308365747, + "tokens_seen": 1692073984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956363636363637, + "loss": 1.3853, + "theoretical_loss": 3.479339022362545, + "tokens_seen": 1692139520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956161616161616, + "loss": 1.5345, + "theoretical_loss": 3.47932741446398, + "tokens_seen": 1692205056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955959595959596, + "loss": 1.6217, + "theoretical_loss": 3.4793158071408277, + "tokens_seen": 1692270592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955757575757576, + "loss": 1.6179, + "theoretical_loss": 3.4793042003930377, + "tokens_seen": 1692336128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955555555555556, + "loss": 1.3264, + "theoretical_loss": 3.479292594220559, + "tokens_seen": 1692401664 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955353535353535, + "loss": 1.6683, + "theoretical_loss": 3.4792809886233416, + "tokens_seen": 1692467200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955151515151515, + "loss": 1.6922, + "theoretical_loss": 3.4792693836013333, + "tokens_seen": 1692532736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954949494949495, + "loss": 1.6503, + "theoretical_loss": 3.4792577791544845, + "tokens_seen": 1692598272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954747474747474, + "loss": 1.6034, + "theoretical_loss": 3.4792461752827446, + "tokens_seen": 1692663808 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.4952957034111023, + "objective/train/avg_token_score": 0.008989947848021984, + "objective/train/docs_used": 959159, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6178122758865356, + "objective/train/original_loss": 3.7790725231170654, + "objective/train/theoretical_loss": 3.479234571986062, + "objective/train/tokens_used": 51588576, + "objective/train/unlikelihood_loss": 0.05506987124681473, + "theoretical_loss": 3.479234571986062, + "tokens_seen": 1692729344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954545454545455, + "loss": 1.5392, + "theoretical_loss": 3.479234571986062, + "tokens_seen": 1692729344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954343434343435, + "loss": 1.5108, + "theoretical_loss": 3.479222969264386, + "tokens_seen": 1692794880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954141414141415, + "loss": 1.404, + "theoretical_loss": 3.4792113671176663, + "tokens_seen": 1692860416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953939393939394, + "loss": 1.4155, + "theoretical_loss": 3.479199765545852, + "tokens_seen": 1692925952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953737373737374, + "loss": 1.4939, + "theoretical_loss": 3.4791881645488916, + "tokens_seen": 1692991488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953535353535354, + "loss": 1.5546, + "theoretical_loss": 3.479176564126736, + "tokens_seen": 1693057024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953333333333334, + "loss": 1.6032, + "theoretical_loss": 3.4791649642793328, + "tokens_seen": 1693122560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953131313131313, + "loss": 1.4701, + "theoretical_loss": 3.479153365006632, + "tokens_seen": 1693188096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952929292929293, + "loss": 1.4542, + "theoretical_loss": 3.479141766308584, + "tokens_seen": 1693253632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952727272727272, + "loss": 1.508, + "theoretical_loss": 3.479130168185136, + "tokens_seen": 1693319168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952525252525252, + "loss": 1.5405, + "theoretical_loss": 3.4791185706362384, + "tokens_seen": 1693384704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952323232323232, + "loss": 1.3952, + "theoretical_loss": 3.4791069736618407, + "tokens_seen": 1693450240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952121212121213, + "loss": 1.4216, + "theoretical_loss": 3.479095377261892, + "tokens_seen": 1693515776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951919191919192, + "loss": 1.5664, + "theoretical_loss": 3.4790837814363407, + "tokens_seen": 1693581312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951717171717172, + "loss": 1.6256, + "theoretical_loss": 3.479072186185138, + "tokens_seen": 1693646848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951515151515152, + "loss": 1.3279, + "theoretical_loss": 3.4790605915082313, + "tokens_seen": 1693712384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951313131313132, + "loss": 1.3493, + "theoretical_loss": 3.479048997405571, + "tokens_seen": 1693777920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951111111111111, + "loss": 1.6102, + "theoretical_loss": 3.479037403877107, + "tokens_seen": 1693843456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950909090909091, + "loss": 1.2949, + "theoretical_loss": 3.4790258109227867, + "tokens_seen": 1693908992 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495070707070707, + "loss": 1.3058, + "theoretical_loss": 3.479014218542561, + "tokens_seen": 1693974528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950505050505051, + "loss": 1.375, + "theoretical_loss": 3.4790026267363796, + "tokens_seen": 1694040064 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495030303030303, + "loss": 1.4453, + "theoretical_loss": 3.478991035504191, + "tokens_seen": 1694105600 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495010101010101, + "loss": 1.55, + "theoretical_loss": 3.4789794448459443, + "tokens_seen": 1694171136 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494989898989899, + "loss": 1.5699, + "theoretical_loss": 3.4789678547615894, + "tokens_seen": 1694236672 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494969696969697, + "loss": 1.3551, + "theoretical_loss": 3.478956265251076, + "tokens_seen": 1694302208 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.3170210123062134, + "objective/train/avg_token_score": 0.017827244475483894, + "objective/train/docs_used": 960536, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9088751673698425, + "objective/train/original_loss": 4.046257972717285, + "objective/train/theoretical_loss": 3.478944676314353, + "objective/train/tokens_used": 53226976, + "objective/train/unlikelihood_loss": 0.10128523409366608, + "theoretical_loss": 3.478944676314353, + "tokens_seen": 1694367744 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494949494949495, + "loss": 1.3764, + "theoretical_loss": 3.478944676314353, + "tokens_seen": 1694367744 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494929292929293, + "loss": 1.7527, + "theoretical_loss": 3.4789330879513694, + "tokens_seen": 1694433280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949090909090909, + "loss": 1.5846, + "theoretical_loss": 3.478921500162076, + "tokens_seen": 1694498816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948888888888889, + "loss": 1.3229, + "theoretical_loss": 3.4789099129464205, + "tokens_seen": 1694564352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948686868686869, + "loss": 1.5288, + "theoretical_loss": 3.4788983263043534, + "tokens_seen": 1694629888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948484848484849, + "loss": 1.5913, + "theoretical_loss": 3.478886740235824, + "tokens_seen": 1694695424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948282828282828, + "loss": 1.4224, + "theoretical_loss": 3.4788751547407815, + "tokens_seen": 1694760960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948080808080808, + "loss": 1.6395, + "theoretical_loss": 3.478863569819176, + "tokens_seen": 1694826496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947878787878787, + "loss": 1.4925, + "theoretical_loss": 3.478851985470956, + "tokens_seen": 1694892032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947676767676767, + "loss": 1.5345, + "theoretical_loss": 3.478840401696071, + "tokens_seen": 1694957568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947474747474748, + "loss": 1.3046, + "theoretical_loss": 3.4788288184944713, + "tokens_seen": 1695023104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947272727272728, + "loss": 1.5731, + "theoretical_loss": 3.4788172358661056, + "tokens_seen": 1695088640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947070707070707, + "loss": 1.3865, + "theoretical_loss": 3.478805653810924, + "tokens_seen": 1695154176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946868686868687, + "loss": 1.3987, + "theoretical_loss": 3.478794072328875, + "tokens_seen": 1695219712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946666666666667, + "loss": 1.4943, + "theoretical_loss": 3.478782491419909, + "tokens_seen": 1695285248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946464646464647, + "loss": 1.2217, + "theoretical_loss": 3.4787709110839753, + "tokens_seen": 1695350784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946262626262626, + "loss": 1.4646, + "theoretical_loss": 3.478759331321023, + "tokens_seen": 1695416320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946060606060606, + "loss": 1.4741, + "theoretical_loss": 3.478747752131002, + "tokens_seen": 1695481856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945858585858586, + "loss": 1.4813, + "theoretical_loss": 3.4787361735138624, + "tokens_seen": 1695547392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945656565656565, + "loss": 1.3267, + "theoretical_loss": 3.478724595469552, + "tokens_seen": 1695612928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945454545454545, + "loss": 1.3645, + "theoretical_loss": 3.4787130179980217, + "tokens_seen": 1695678464 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945252525252526, + "loss": 1.4331, + "theoretical_loss": 3.478701441099221, + "tokens_seen": 1695744000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945050505050506, + "loss": 1.481, + "theoretical_loss": 3.4786898647730986, + "tokens_seen": 1695809536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944848484848485, + "loss": 1.4608, + "theoretical_loss": 3.478678289019605, + "tokens_seen": 1695875072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944646464646465, + "loss": 1.4611, + "theoretical_loss": 3.478666713838689, + "tokens_seen": 1695940608 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.3928396999835968, + "objective/train/avg_token_score": 0.021624328568577766, + "objective/train/docs_used": 961177, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3411294221878052, + "objective/train/original_loss": 4.00459623336792, + "objective/train/theoretical_loss": 3.478655139230301, + "objective/train/tokens_used": 54865376, + "objective/train/unlikelihood_loss": 0.05852148309350014, + "theoretical_loss": 3.478655139230301, + "tokens_seen": 1696006144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944444444444445, + "loss": 1.4319, + "theoretical_loss": 3.478655139230301, + "tokens_seen": 1696006144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944242424242424, + "loss": 1.432, + "theoretical_loss": 3.478643565194389, + "tokens_seen": 1696071680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944040404040404, + "loss": 1.4987, + "theoretical_loss": 3.478631991730904, + "tokens_seen": 1696137216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943838383838384, + "loss": 1.6415, + "theoretical_loss": 3.478620418839795, + "tokens_seen": 1696202752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943636363636363, + "loss": 1.6562, + "theoretical_loss": 3.4786088465210123, + "tokens_seen": 1696268288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943434343434343, + "loss": 1.3873, + "theoretical_loss": 3.478597274774504, + "tokens_seen": 1696333824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943232323232323, + "loss": 1.3246, + "theoretical_loss": 3.4785857036002215, + "tokens_seen": 1696399360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943030303030303, + "loss": 1.3636, + "theoretical_loss": 3.478574132998113, + "tokens_seen": 1696464896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942828282828283, + "loss": 1.446, + "theoretical_loss": 3.478562562968129, + "tokens_seen": 1696530432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942626262626263, + "loss": 1.3962, + "theoretical_loss": 3.4785509935102183, + "tokens_seen": 1696595968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942424242424243, + "loss": 1.4457, + "theoretical_loss": 3.4785394246243317, + "tokens_seen": 1696661504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942222222222223, + "loss": 1.6089, + "theoretical_loss": 3.478527856310417, + "tokens_seen": 1696727040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942020202020202, + "loss": 1.5574, + "theoretical_loss": 3.4785162885684255, + "tokens_seen": 1696792576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941818181818182, + "loss": 1.4105, + "theoretical_loss": 3.478504721398306, + "tokens_seen": 1696858112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941616161616161, + "loss": 1.399, + "theoretical_loss": 3.4784931548000086, + "tokens_seen": 1696923648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941414141414142, + "loss": 1.5323, + "theoretical_loss": 3.4784815887734823, + "tokens_seen": 1696989184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941212121212121, + "loss": 1.5829, + "theoretical_loss": 3.4784700233186774, + "tokens_seen": 1697054720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941010101010101, + "loss": 1.4786, + "theoretical_loss": 3.4784584584355436, + "tokens_seen": 1697120256 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494080808080808, + "loss": 1.5934, + "theoretical_loss": 3.47844689412403, + "tokens_seen": 1697185792 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494060606060606, + "loss": 1.4969, + "theoretical_loss": 3.4784353303840865, + "tokens_seen": 1697251328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940404040404041, + "loss": 1.5708, + "theoretical_loss": 3.478423767215663, + "tokens_seen": 1697316864 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940202020202021, + "loss": 1.3845, + "theoretical_loss": 3.478412204618709, + "tokens_seen": 1697382400 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494, + "loss": 1.3359, + "theoretical_loss": 3.4784006425931744, + "tokens_seen": 1697447936 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493979797979798, + "loss": 1.4839, + "theoretical_loss": 3.478389081139009, + "tokens_seen": 1697513472 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493959595959596, + "loss": 1.5483, + "theoretical_loss": 3.4783775202561618, + "tokens_seen": 1697579008 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.5273093581199646, + "objective/train/avg_token_score": 0.005273816175758839, + "objective/train/docs_used": 961828, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.706133484840393, + "objective/train/original_loss": 3.877432107925415, + "objective/train/theoretical_loss": 3.4783659599445826, + "objective/train/tokens_used": 56503776, + "objective/train/unlikelihood_loss": 0.0712694600224495, + "theoretical_loss": 3.4783659599445826, + "tokens_seen": 1697644544 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493939393939394, + "loss": 1.5571, + "theoretical_loss": 3.4783659599445826, + "tokens_seen": 1697644544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939191919191919, + "loss": 1.2926, + "theoretical_loss": 3.478354400204222, + "tokens_seen": 1697710080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938989898989899, + "loss": 1.3542, + "theoretical_loss": 3.4783428410350297, + "tokens_seen": 1697775616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938787878787878, + "loss": 1.6162, + "theoretical_loss": 3.4783312824369537, + "tokens_seen": 1697841152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938585858585859, + "loss": 1.5232, + "theoretical_loss": 3.478319724409946, + "tokens_seen": 1697906688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938383838383838, + "loss": 1.5115, + "theoretical_loss": 3.478308166953955, + "tokens_seen": 1697972224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938181818181819, + "loss": 1.6203, + "theoretical_loss": 3.4782966100689308, + "tokens_seen": 1698037760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937979797979798, + "loss": 1.6335, + "theoretical_loss": 3.4782850537548233, + "tokens_seen": 1698103296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937777777777778, + "loss": 1.3283, + "theoretical_loss": 3.478273498011582, + "tokens_seen": 1698168832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937575757575758, + "loss": 1.5045, + "theoretical_loss": 3.4782619428391564, + "tokens_seen": 1698234368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937373737373738, + "loss": 1.5789, + "theoretical_loss": 3.478250388237497, + "tokens_seen": 1698299904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937171717171717, + "loss": 1.4186, + "theoretical_loss": 3.478238834206553, + "tokens_seen": 1698365440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936969696969697, + "loss": 1.4197, + "theoretical_loss": 3.4782272807462746, + "tokens_seen": 1698430976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936767676767677, + "loss": 1.384, + "theoretical_loss": 3.4782157278566115, + "tokens_seen": 1698496512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936565656565657, + "loss": 1.553, + "theoretical_loss": 3.4782041755375133, + "tokens_seen": 1698562048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936363636363636, + "loss": 1.4178, + "theoretical_loss": 3.47819262378893, + "tokens_seen": 1698627584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936161616161616, + "loss": 1.6003, + "theoretical_loss": 3.478181072610811, + "tokens_seen": 1698693120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935959595959597, + "loss": 1.0584, + "theoretical_loss": 3.478169522003107, + "tokens_seen": 1698758656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935757575757576, + "loss": 1.4205, + "theoretical_loss": 3.478157971965767, + "tokens_seen": 1698824192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935555555555556, + "loss": 1.4223, + "theoretical_loss": 3.4781464224987415, + "tokens_seen": 1698889728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935353535353536, + "loss": 1.3599, + "theoretical_loss": 3.478134873601979, + "tokens_seen": 1698955264 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935151515151515, + "loss": 1.375, + "theoretical_loss": 3.4781233252754316, + "tokens_seen": 1699020800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934949494949495, + "loss": 1.4275, + "theoretical_loss": 3.4781117775190467, + "tokens_seen": 1699086336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934747474747475, + "loss": 1.3522, + "theoretical_loss": 3.4781002303327764, + "tokens_seen": 1699151872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934545454545455, + "loss": 1.48, + "theoretical_loss": 3.4780886837165688, + "tokens_seen": 1699217408 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.35807672142982483, + "objective/train/avg_token_score": 0.012293923646211624, + "objective/train/docs_used": 963095, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1824865341186523, + "objective/train/original_loss": 4.2231035232543945, + "objective/train/theoretical_loss": 3.4780771376703745, + "objective/train/tokens_used": 58142176, + "objective/train/unlikelihood_loss": 0.0799039676785469, + "theoretical_loss": 3.4780771376703745, + "tokens_seen": 1699282944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934343434343434, + "loss": 1.2993, + "theoretical_loss": 3.4780771376703745, + "tokens_seen": 1699282944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934141414141414, + "loss": 1.4704, + "theoretical_loss": 3.4780655921941435, + "tokens_seen": 1699348480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933939393939394, + "loss": 1.3564, + "theoretical_loss": 3.4780540472878254, + "tokens_seen": 1699414016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933737373737373, + "loss": 1.487, + "theoretical_loss": 3.47804250295137, + "tokens_seen": 1699479552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933535353535354, + "loss": 1.331, + "theoretical_loss": 3.478030959184728, + "tokens_seen": 1699545088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933333333333334, + "loss": 1.6139, + "theoretical_loss": 3.4780194159878484, + "tokens_seen": 1699610624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933131313131314, + "loss": 1.4798, + "theoretical_loss": 3.4780078733606814, + "tokens_seen": 1699676160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932929292929293, + "loss": 1.516, + "theoretical_loss": 3.477996331303177, + "tokens_seen": 1699741696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932727272727273, + "loss": 1.5999, + "theoretical_loss": 3.4779847898152854, + "tokens_seen": 1699807232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932525252525253, + "loss": 1.4377, + "theoretical_loss": 3.477973248896956, + "tokens_seen": 1699872768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932323232323232, + "loss": 1.4555, + "theoretical_loss": 3.4779617085481385, + "tokens_seen": 1699938304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932121212121212, + "loss": 1.2517, + "theoretical_loss": 3.4779501687687837, + "tokens_seen": 1700003840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931919191919192, + "loss": 1.5707, + "theoretical_loss": 3.477938629558841, + "tokens_seen": 1700069376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931717171717171, + "loss": 1.4201, + "theoretical_loss": 3.4779270909182607, + "tokens_seen": 1700134912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931515151515151, + "loss": 1.539, + "theoretical_loss": 3.4779155528469925, + "tokens_seen": 1700200448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931313131313131, + "loss": 1.4055, + "theoretical_loss": 3.477904015344986, + "tokens_seen": 1700265984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931111111111112, + "loss": 1.628, + "theoretical_loss": 3.4778924784121914, + "tokens_seen": 1700331520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930909090909091, + "loss": 1.4366, + "theoretical_loss": 3.47788094204856, + "tokens_seen": 1700397056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930707070707071, + "loss": 1.5349, + "theoretical_loss": 3.4778694062540394, + "tokens_seen": 1700462592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930505050505051, + "loss": 1.335, + "theoretical_loss": 3.4778578710285815, + "tokens_seen": 1700528128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930303030303031, + "loss": 1.4479, + "theoretical_loss": 3.477846336372136, + "tokens_seen": 1700593664 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493010101010101, + "loss": 1.6763, + "theoretical_loss": 3.4778348022846517, + "tokens_seen": 1700659200 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492989898989899, + "loss": 1.5868, + "theoretical_loss": 3.47782326876608, + "tokens_seen": 1700724736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929696969696969, + "loss": 1.4506, + "theoretical_loss": 3.4778117358163705, + "tokens_seen": 1700790272 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492949494949495, + "loss": 1.5949, + "theoretical_loss": 3.4778002034354722, + "tokens_seen": 1700855808 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.4352394938468933, + "objective/train/avg_token_score": 0.022302182391285896, + "objective/train/docs_used": 963811, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3806675672531128, + "objective/train/original_loss": 3.8218584060668945, + "objective/train/theoretical_loss": 3.477788671623337, + "objective/train/tokens_used": 59780576, + "objective/train/unlikelihood_loss": 0.0842813178896904, + "theoretical_loss": 3.477788671623337, + "tokens_seen": 1700921344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929292929292929, + "loss": 1.6276, + "theoretical_loss": 3.477788671623337, + "tokens_seen": 1700921344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929090909090909, + "loss": 1.3813, + "theoretical_loss": 3.4777771403799136, + "tokens_seen": 1700986880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928888888888888, + "loss": 1.6226, + "theoretical_loss": 3.4777656097051524, + "tokens_seen": 1701052416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928686868686869, + "loss": 1.6444, + "theoretical_loss": 3.4777540795990034, + "tokens_seen": 1701117952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928484848484849, + "loss": 1.3509, + "theoretical_loss": 3.477742550061417, + "tokens_seen": 1701183488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928282828282829, + "loss": 1.2636, + "theoretical_loss": 3.4777310210923424, + "tokens_seen": 1701249024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928080808080808, + "loss": 1.486, + "theoretical_loss": 3.4777194926917305, + "tokens_seen": 1701314560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927878787878788, + "loss": 1.339, + "theoretical_loss": 3.4777079648595315, + "tokens_seen": 1701380096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927676767676768, + "loss": 1.5191, + "theoretical_loss": 3.4776964375956947, + "tokens_seen": 1701445632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927474747474748, + "loss": 1.4387, + "theoretical_loss": 3.477684910900171, + "tokens_seen": 1701511168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927272727272727, + "loss": 1.3134, + "theoretical_loss": 3.4776733847729098, + "tokens_seen": 1701576704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927070707070707, + "loss": 1.4071, + "theoretical_loss": 3.4776618592138613, + "tokens_seen": 1701642240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926868686868686, + "loss": 1.3664, + "theoretical_loss": 3.477650334222976, + "tokens_seen": 1701707776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926666666666666, + "loss": 1.5606, + "theoretical_loss": 3.477638809800204, + "tokens_seen": 1701773312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926464646464647, + "loss": 1.4081, + "theoretical_loss": 3.477627285945495, + "tokens_seen": 1701838848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926262626262627, + "loss": 1.5566, + "theoretical_loss": 3.4776157626587993, + "tokens_seen": 1701904384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926060606060606, + "loss": 1.3377, + "theoretical_loss": 3.4776042399400673, + "tokens_seen": 1701969920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925858585858586, + "loss": 1.1917, + "theoretical_loss": 3.477592717789248, + "tokens_seen": 1702035456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925656565656566, + "loss": 1.3057, + "theoretical_loss": 3.4775811962062937, + "tokens_seen": 1702100992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925454545454546, + "loss": 1.4049, + "theoretical_loss": 3.4775696751911527, + "tokens_seen": 1702166528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925252525252525, + "loss": 1.5844, + "theoretical_loss": 3.4775581547437757, + "tokens_seen": 1702232064 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925050505050505, + "loss": 1.5742, + "theoretical_loss": 3.477546634864113, + "tokens_seen": 1702297600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924848484848485, + "loss": 1.6743, + "theoretical_loss": 3.477535115552115, + "tokens_seen": 1702363136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924646464646464, + "loss": 1.7692, + "theoretical_loss": 3.4775235968077314, + "tokens_seen": 1702428672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924444444444444, + "loss": 1.4189, + "theoretical_loss": 3.4775120786309124, + "tokens_seen": 1702494208 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.4039589464664459, + "objective/train/avg_token_score": 0.022879842668771744, + "objective/train/docs_used": 965136, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2613023519515991, + "objective/train/original_loss": 4.1900811195373535, + "objective/train/theoretical_loss": 3.4775005610216088, + "objective/train/tokens_used": 61418976, + "objective/train/unlikelihood_loss": 0.07926816493272781, + "theoretical_loss": 3.4775005610216088, + "tokens_seen": 1702559744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004924242424242425, + "loss": 1.3197, + "theoretical_loss": 3.4775005610216088, + "tokens_seen": 1702559744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004924040404040405, + "loss": 1.4524, + "theoretical_loss": 3.4774890439797694, + "tokens_seen": 1702625280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923838383838384, + "loss": 1.4616, + "theoretical_loss": 3.4774775275053464, + "tokens_seen": 1702690816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923636363636364, + "loss": 1.4173, + "theoretical_loss": 3.4774660115982883, + "tokens_seen": 1702756352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923434343434344, + "loss": 1.4959, + "theoretical_loss": 3.477454496258546, + "tokens_seen": 1702821888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923232323232323, + "loss": 1.4659, + "theoretical_loss": 3.47744298148607, + "tokens_seen": 1702887424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923030303030303, + "loss": 1.4347, + "theoretical_loss": 3.4774314672808098, + "tokens_seen": 1702952960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922828282828283, + "loss": 1.4188, + "theoretical_loss": 3.477419953642716, + "tokens_seen": 1703018496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922626262626262, + "loss": 1.4657, + "theoretical_loss": 3.477408440571739, + "tokens_seen": 1703084032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922424242424242, + "loss": 1.2297, + "theoretical_loss": 3.4773969280678294, + "tokens_seen": 1703149568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922222222222222, + "loss": 1.2242, + "theoretical_loss": 3.4773854161309363, + "tokens_seen": 1703215104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922020202020202, + "loss": 1.3182, + "theoretical_loss": 3.477373904761011, + "tokens_seen": 1703280640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921818181818182, + "loss": 1.4721, + "theoretical_loss": 3.4773623939580034, + "tokens_seen": 1703346176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921616161616162, + "loss": 1.7844, + "theoretical_loss": 3.477350883721863, + "tokens_seen": 1703411712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921414141414142, + "loss": 1.3457, + "theoretical_loss": 3.477339374052542, + "tokens_seen": 1703477248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921212121212122, + "loss": 1.3502, + "theoretical_loss": 3.477327864949989, + "tokens_seen": 1703542784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921010101010101, + "loss": 1.6302, + "theoretical_loss": 3.4773163564141547, + "tokens_seen": 1703608320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920808080808081, + "loss": 1.3801, + "theoretical_loss": 3.4773048484449895, + "tokens_seen": 1703673856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920606060606061, + "loss": 1.4358, + "theoretical_loss": 3.4772933410424436, + "tokens_seen": 1703739392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920404040404041, + "loss": 1.7846, + "theoretical_loss": 3.477281834206468, + "tokens_seen": 1703804928 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492020202020202, + "loss": 1.4055, + "theoretical_loss": 3.4772703279370116, + "tokens_seen": 1703870464 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492, + "loss": 1.5334, + "theoretical_loss": 3.477258822234026, + "tokens_seen": 1703936000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919797979797979, + "loss": 1.5056, + "theoretical_loss": 3.4772473170974614, + "tokens_seen": 1704001536 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491959595959596, + "loss": 1.1744, + "theoretical_loss": 3.477235812527267, + "tokens_seen": 1704067072 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491939393939394, + "loss": 1.5387, + "theoretical_loss": 3.4772243085233945, + "tokens_seen": 1704132608 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.3712732195854187, + "objective/train/avg_token_score": 0.018682070076465607, + "objective/train/docs_used": 965825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2413780689239502, + "objective/train/original_loss": 4.305376052856445, + "objective/train/theoretical_loss": 3.4772128050857933, + "objective/train/tokens_used": 63057376, + "objective/train/unlikelihood_loss": 0.054906729608774185, + "theoretical_loss": 3.4772128050857933, + "tokens_seen": 1704198144 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491919191919192, + "loss": 1.4768, + "theoretical_loss": 3.4772128050857933, + "tokens_seen": 1704198144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918989898989899, + "loss": 1.5636, + "theoretical_loss": 3.4772013022144144, + "tokens_seen": 1704263680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918787878787879, + "loss": 1.3053, + "theoretical_loss": 3.4771897999092083, + "tokens_seen": 1704329216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918585858585859, + "loss": 1.5969, + "theoretical_loss": 3.4771782981701245, + "tokens_seen": 1704394752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918383838383839, + "loss": 1.5039, + "theoretical_loss": 3.4771667969971136, + "tokens_seen": 1704460288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918181818181818, + "loss": 1.2291, + "theoretical_loss": 3.4771552963901264, + "tokens_seen": 1704525824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917979797979798, + "loss": 1.3308, + "theoretical_loss": 3.477143796349113, + "tokens_seen": 1704591360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917777777777777, + "loss": 1.4503, + "theoretical_loss": 3.477132296874024, + "tokens_seen": 1704656896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917575757575758, + "loss": 1.4605, + "theoretical_loss": 3.47712079796481, + "tokens_seen": 1704722432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917373737373737, + "loss": 1.553, + "theoretical_loss": 3.4771092996214206, + "tokens_seen": 1704787968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917171717171718, + "loss": 1.5346, + "theoretical_loss": 3.4770978018438066, + "tokens_seen": 1704853504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916969696969697, + "loss": 1.5756, + "theoretical_loss": 3.4770863046319187, + "tokens_seen": 1704919040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916767676767677, + "loss": 1.4356, + "theoretical_loss": 3.4770748079857072, + "tokens_seen": 1704984576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916565656565657, + "loss": 1.441, + "theoretical_loss": 3.4770633119051224, + "tokens_seen": 1705050112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916363636363637, + "loss": 1.3771, + "theoretical_loss": 3.4770518163901145, + "tokens_seen": 1705115648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916161616161616, + "loss": 1.4827, + "theoretical_loss": 3.4770403214406347, + "tokens_seen": 1705181184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915959595959596, + "loss": 1.2737, + "theoretical_loss": 3.4770288270566327, + "tokens_seen": 1705246720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915757575757576, + "loss": 1.6622, + "theoretical_loss": 3.477017333238059, + "tokens_seen": 1705312256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915555555555556, + "loss": 1.3899, + "theoretical_loss": 3.4770058399848645, + "tokens_seen": 1705377792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915353535353535, + "loss": 1.4385, + "theoretical_loss": 3.476994347296999, + "tokens_seen": 1705443328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915151515151515, + "loss": 1.3491, + "theoretical_loss": 3.476982855174414, + "tokens_seen": 1705508864 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914949494949494, + "loss": 1.4771, + "theoretical_loss": 3.476971363617059, + "tokens_seen": 1705574400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914747474747475, + "loss": 1.3391, + "theoretical_loss": 3.476959872624885, + "tokens_seen": 1705639936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914545454545455, + "loss": 1.4106, + "theoretical_loss": 3.4769483821978424, + "tokens_seen": 1705705472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914343434343435, + "loss": 1.2723, + "theoretical_loss": 3.4769368923358814, + "tokens_seen": 1705771008 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.470796674489975, + "objective/train/avg_token_score": 0.008710848167538643, + "objective/train/docs_used": 966739, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.535874843597412, + "objective/train/original_loss": 3.7179017066955566, + "objective/train/theoretical_loss": 3.476925403038953, + "objective/train/tokens_used": 64695776, + "objective/train/unlikelihood_loss": 0.08725922554731369, + "theoretical_loss": 3.476925403038953, + "tokens_seen": 1705836544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914141414141414, + "loss": 1.7468, + "theoretical_loss": 3.476925403038953, + "tokens_seen": 1705836544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913939393939394, + "loss": 1.2985, + "theoretical_loss": 3.4769139143070076, + "tokens_seen": 1705902080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913737373737374, + "loss": 1.3906, + "theoretical_loss": 3.476902426139995, + "tokens_seen": 1705967616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913535353535354, + "loss": 1.5775, + "theoretical_loss": 3.4768909385378666, + "tokens_seen": 1706033152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913333333333333, + "loss": 1.3863, + "theoretical_loss": 3.4768794515005723, + "tokens_seen": 1706098688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913131313131313, + "loss": 1.3093, + "theoretical_loss": 3.476867965028063, + "tokens_seen": 1706164224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912929292929293, + "loss": 1.4804, + "theoretical_loss": 3.47685647912029, + "tokens_seen": 1706229760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912727272727272, + "loss": 1.4999, + "theoretical_loss": 3.476844993777202, + "tokens_seen": 1706295296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912525252525252, + "loss": 1.4509, + "theoretical_loss": 3.476833508998751, + "tokens_seen": 1706360832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912323232323233, + "loss": 1.3798, + "theoretical_loss": 3.4768220247848873, + "tokens_seen": 1706426368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912121212121213, + "loss": 1.6698, + "theoretical_loss": 3.4768105411355608, + "tokens_seen": 1706491904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911919191919192, + "loss": 1.5185, + "theoretical_loss": 3.476799058050723, + "tokens_seen": 1706557440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911717171717172, + "loss": 1.1568, + "theoretical_loss": 3.4767875755303237, + "tokens_seen": 1706622976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911515151515152, + "loss": 1.0498, + "theoretical_loss": 3.476776093574314, + "tokens_seen": 1706688512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911313131313131, + "loss": 1.3549, + "theoretical_loss": 3.4767646121826443, + "tokens_seen": 1706754048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911111111111111, + "loss": 1.4073, + "theoretical_loss": 3.4767531313552658, + "tokens_seen": 1706819584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910909090909091, + "loss": 1.3959, + "theoretical_loss": 3.4767416510921274, + "tokens_seen": 1706885120 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491070707070707, + "loss": 1.2292, + "theoretical_loss": 3.4767301713931813, + "tokens_seen": 1706950656 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491050505050505, + "loss": 1.4875, + "theoretical_loss": 3.4767186922583777, + "tokens_seen": 1707016192 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491030303030303, + "loss": 1.3406, + "theoretical_loss": 3.4767072136876673, + "tokens_seen": 1707081728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910101010101011, + "loss": 1.3963, + "theoretical_loss": 3.4766957356810004, + "tokens_seen": 1707147264 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490989898989899, + "loss": 1.3853, + "theoretical_loss": 3.4766842582383277, + "tokens_seen": 1707212800 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490969696969697, + "loss": 1.2996, + "theoretical_loss": 3.4766727813596, + "tokens_seen": 1707278336 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490949494949495, + "loss": 1.6414, + "theoretical_loss": 3.476661305044768, + "tokens_seen": 1707343872 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490929292929293, + "loss": 1.3811, + "theoretical_loss": 3.476649829293782, + "tokens_seen": 1707409408 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.4374389052391052, + "objective/train/avg_token_score": 0.005900775548070669, + "objective/train/docs_used": 967356, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5339224338531494, + "objective/train/original_loss": 4.014285087585449, + "objective/train/theoretical_loss": 3.4766383541065933, + "objective/train/tokens_used": 66334176, + "objective/train/unlikelihood_loss": 0.08221994340419769, + "theoretical_loss": 3.4766383541065933, + "tokens_seen": 1707474944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909090909090909, + "loss": 1.4133, + "theoretical_loss": 3.4766383541065933, + "tokens_seen": 1707474944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908888888888889, + "loss": 1.2069, + "theoretical_loss": 3.4766268794831516, + "tokens_seen": 1707540480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908686868686868, + "loss": 1.2704, + "theoretical_loss": 3.4766154054234084, + "tokens_seen": 1707606016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908484848484849, + "loss": 1.5368, + "theoretical_loss": 3.4766039319273143, + "tokens_seen": 1707671552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908282828282828, + "loss": 1.2068, + "theoretical_loss": 3.4765924589948196, + "tokens_seen": 1707737088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908080808080808, + "loss": 1.3801, + "theoretical_loss": 3.4765809866258754, + "tokens_seen": 1707802624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907878787878787, + "loss": 1.5021, + "theoretical_loss": 3.476569514820432, + "tokens_seen": 1707868160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907676767676768, + "loss": 1.599, + "theoretical_loss": 3.4765580435784402, + "tokens_seen": 1707933696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907474747474748, + "loss": 1.6612, + "theoretical_loss": 3.4765465728998506, + "tokens_seen": 1707999232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907272727272728, + "loss": 1.7327, + "theoretical_loss": 3.476535102784614, + "tokens_seen": 1708064768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907070707070707, + "loss": 1.3258, + "theoretical_loss": 3.4765236332326817, + "tokens_seen": 1708130304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906868686868687, + "loss": 1.2935, + "theoretical_loss": 3.4765121642440038, + "tokens_seen": 1708195840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906666666666667, + "loss": 1.477, + "theoretical_loss": 3.476500695818531, + "tokens_seen": 1708261376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906464646464647, + "loss": 1.5143, + "theoretical_loss": 3.476489227956214, + "tokens_seen": 1708326912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906262626262626, + "loss": 1.5016, + "theoretical_loss": 3.476477760657004, + "tokens_seen": 1708392448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906060606060606, + "loss": 1.3412, + "theoretical_loss": 3.4764662939208515, + "tokens_seen": 1708457984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905858585858585, + "loss": 1.3278, + "theoretical_loss": 3.476454827747707, + "tokens_seen": 1708523520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905656565656565, + "loss": 1.4474, + "theoretical_loss": 3.476443362137522, + "tokens_seen": 1708589056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905454545454546, + "loss": 1.3127, + "theoretical_loss": 3.4764318970902464, + "tokens_seen": 1708654592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905252525252526, + "loss": 1.5852, + "theoretical_loss": 3.4764204326058317, + "tokens_seen": 1708720128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905050505050505, + "loss": 1.328, + "theoretical_loss": 3.476408968684228, + "tokens_seen": 1708785664 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904848484848485, + "loss": 1.5521, + "theoretical_loss": 3.4763975053253864, + "tokens_seen": 1708851200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904646464646465, + "loss": 1.4926, + "theoretical_loss": 3.4763860425292576, + "tokens_seen": 1708916736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904444444444445, + "loss": 1.3039, + "theoretical_loss": 3.4763745802957926, + "tokens_seen": 1708982272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904242424242424, + "loss": 1.5436, + "theoretical_loss": 3.476363118624943, + "tokens_seen": 1709047808 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.4849095940589905, + "objective/train/avg_token_score": 0.0028560131322592497, + "objective/train/docs_used": 968332, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6418017148971558, + "objective/train/original_loss": 3.7804646492004395, + "objective/train/theoretical_loss": 3.4763516575166573, + "objective/train/tokens_used": 67972576, + "objective/train/unlikelihood_loss": 0.1123645082116127, + "theoretical_loss": 3.4763516575166573, + "tokens_seen": 1709113344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904040404040404, + "loss": 1.516, + "theoretical_loss": 3.4763516575166573, + "tokens_seen": 1709113344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903838383838384, + "loss": 1.3689, + "theoretical_loss": 3.476340196970889, + "tokens_seen": 1709178880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903636363636364, + "loss": 1.6257, + "theoretical_loss": 3.476328736987587, + "tokens_seen": 1709244416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903434343434343, + "loss": 1.3624, + "theoretical_loss": 3.476317277566703, + "tokens_seen": 1709309952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903232323232323, + "loss": 1.4273, + "theoretical_loss": 3.4763058187081874, + "tokens_seen": 1709375488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903030303030304, + "loss": 1.3265, + "theoretical_loss": 3.476294360411991, + "tokens_seen": 1709441024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902828282828283, + "loss": 1.6407, + "theoretical_loss": 3.4762829026780655, + "tokens_seen": 1709506560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902626262626263, + "loss": 1.5641, + "theoretical_loss": 3.4762714455063612, + "tokens_seen": 1709572096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902424242424243, + "loss": 1.4669, + "theoretical_loss": 3.4762599888968286, + "tokens_seen": 1709637632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902222222222222, + "loss": 1.4743, + "theoretical_loss": 3.476248532849419, + "tokens_seen": 1709703168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902020202020202, + "loss": 1.1262, + "theoretical_loss": 3.4762370773640834, + "tokens_seen": 1709768704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901818181818182, + "loss": 1.4312, + "theoretical_loss": 3.476225622440772, + "tokens_seen": 1709834240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901616161616162, + "loss": 1.3713, + "theoretical_loss": 3.4762141680794367, + "tokens_seen": 1709899776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901414141414141, + "loss": 1.4197, + "theoretical_loss": 3.476202714280028, + "tokens_seen": 1709965312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901212121212121, + "loss": 1.3616, + "theoretical_loss": 3.476191261042496, + "tokens_seen": 1710030848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901010101010101, + "loss": 1.3839, + "theoretical_loss": 3.4761798083667923, + "tokens_seen": 1710096384 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490080808080808, + "loss": 1.6839, + "theoretical_loss": 3.476168356252868, + "tokens_seen": 1710161920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900606060606061, + "loss": 1.3635, + "theoretical_loss": 3.476156904700674, + "tokens_seen": 1710227456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900404040404041, + "loss": 1.6225, + "theoretical_loss": 3.476145453710161, + "tokens_seen": 1710292992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900202020202021, + "loss": 1.3887, + "theoretical_loss": 3.4761340032812793, + "tokens_seen": 1710358528 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049, + "loss": 1.4664, + "theoretical_loss": 3.4761225534139806, + "tokens_seen": 1710424064 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489979797979798, + "loss": 1.388, + "theoretical_loss": 3.476111104108216, + "tokens_seen": 1710489600 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489959595959596, + "loss": 1.4894, + "theoretical_loss": 3.476099655363936, + "tokens_seen": 1710555136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899393939393939, + "loss": 1.2687, + "theoretical_loss": 3.476088207181092, + "tokens_seen": 1710620672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899191919191919, + "loss": 1.298, + "theoretical_loss": 3.4760767595596347, + "tokens_seen": 1710686208 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.5306084752082825, + "objective/train/avg_token_score": 0.01167221274226904, + "objective/train/docs_used": 968886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6835694313049316, + "objective/train/original_loss": 4.055310249328613, + "objective/train/theoretical_loss": 3.4760653124995144, + "objective/train/tokens_used": 69610976, + "objective/train/unlikelihood_loss": 0.05426410958170891, + "theoretical_loss": 3.4760653124995144, + "tokens_seen": 1710751744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898989898989899, + "loss": 1.7209, + "theoretical_loss": 3.4760653124995144, + "tokens_seen": 1710751744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898787878787878, + "loss": 1.2871, + "theoretical_loss": 3.4760538660006834, + "tokens_seen": 1710817280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898585858585858, + "loss": 1.4492, + "theoretical_loss": 3.4760424200630915, + "tokens_seen": 1710882816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898383838383839, + "loss": 1.4377, + "theoretical_loss": 3.4760309746866906, + "tokens_seen": 1710948352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898181818181819, + "loss": 1.4232, + "theoretical_loss": 3.476019529871431, + "tokens_seen": 1711013888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897979797979798, + "loss": 1.6333, + "theoretical_loss": 3.4760080856172637, + "tokens_seen": 1711079424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897777777777778, + "loss": 1.4597, + "theoretical_loss": 3.475996641924141, + "tokens_seen": 1711144960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897575757575758, + "loss": 1.5103, + "theoretical_loss": 3.475985198792012, + "tokens_seen": 1711210496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897373737373738, + "loss": 1.3766, + "theoretical_loss": 3.4759737562208284, + "tokens_seen": 1711276032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897171717171717, + "loss": 1.532, + "theoretical_loss": 3.4759623142105416, + "tokens_seen": 1711341568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896969696969697, + "loss": 1.4063, + "theoretical_loss": 3.4759508727611026, + "tokens_seen": 1711407104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896767676767676, + "loss": 1.3932, + "theoretical_loss": 3.475939431872462, + "tokens_seen": 1711472640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896565656565657, + "loss": 1.4003, + "theoretical_loss": 3.4759279915445713, + "tokens_seen": 1711538176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896363636363636, + "loss": 1.335, + "theoretical_loss": 3.4759165517773813, + "tokens_seen": 1711603712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896161616161617, + "loss": 1.1782, + "theoretical_loss": 3.4759051125708433, + "tokens_seen": 1711669248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895959595959596, + "loss": 1.2264, + "theoretical_loss": 3.475893673924908, + "tokens_seen": 1711734784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895757575757576, + "loss": 1.4453, + "theoretical_loss": 3.475882235839527, + "tokens_seen": 1711800320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895555555555556, + "loss": 1.4011, + "theoretical_loss": 3.4758707983146504, + "tokens_seen": 1711865856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895353535353536, + "loss": 1.4299, + "theoretical_loss": 3.47585936135023, + "tokens_seen": 1711931392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895151515151515, + "loss": 1.5037, + "theoretical_loss": 3.475847924946217, + "tokens_seen": 1711996928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894949494949495, + "loss": 1.5703, + "theoretical_loss": 3.475836489102562, + "tokens_seen": 1712062464 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894747474747475, + "loss": 1.4185, + "theoretical_loss": 3.4758250538192166, + "tokens_seen": 1712128000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894545454545455, + "loss": 1.6266, + "theoretical_loss": 3.4758136190961313, + "tokens_seen": 1712193536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894343434343434, + "loss": 1.6168, + "theoretical_loss": 3.475802184933258, + "tokens_seen": 1712259072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894141414141414, + "loss": 1.2173, + "theoretical_loss": 3.475790751330547, + "tokens_seen": 1712324608 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.4318181872367859, + "objective/train/avg_token_score": 0.056178782135248184, + "objective/train/docs_used": 970327, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3246653079986572, + "objective/train/original_loss": 3.9383597373962402, + "objective/train/theoretical_loss": 3.47577931828795, + "objective/train/tokens_used": 71249376, + "objective/train/unlikelihood_loss": 0.07031881809234619, + "theoretical_loss": 3.47577931828795, + "tokens_seen": 1712390144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893939393939393, + "loss": 1.4182, + "theoretical_loss": 3.47577931828795, + "tokens_seen": 1712390144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893737373737374, + "loss": 1.1773, + "theoretical_loss": 3.4757678858054177, + "tokens_seen": 1712455680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893535353535354, + "loss": 1.624, + "theoretical_loss": 3.4757564538829016, + "tokens_seen": 1712521216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893333333333334, + "loss": 1.4873, + "theoretical_loss": 3.4757450225203526, + "tokens_seen": 1712586752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893131313131313, + "loss": 1.3469, + "theoretical_loss": 3.475733591717722, + "tokens_seen": 1712652288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892929292929293, + "loss": 1.4407, + "theoretical_loss": 3.4757221614749607, + "tokens_seen": 1712717824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892727272727273, + "loss": 1.4212, + "theoretical_loss": 3.47571073179202, + "tokens_seen": 1712783360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892525252525253, + "loss": 1.3738, + "theoretical_loss": 3.4756993026688514, + "tokens_seen": 1712848896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892323232323232, + "loss": 1.3697, + "theoretical_loss": 3.475687874105406, + "tokens_seen": 1712914432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892121212121212, + "loss": 1.5092, + "theoretical_loss": 3.4756764461016347, + "tokens_seen": 1712979968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891919191919192, + "loss": 1.2167, + "theoretical_loss": 3.4756650186574882, + "tokens_seen": 1713045504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891717171717171, + "loss": 1.4599, + "theoretical_loss": 3.475653591772918, + "tokens_seen": 1713111040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891515151515151, + "loss": 1.61, + "theoretical_loss": 3.4756421654478764, + "tokens_seen": 1713176576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891313131313132, + "loss": 1.679, + "theoretical_loss": 3.475630739682313, + "tokens_seen": 1713242112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891111111111112, + "loss": 1.3736, + "theoretical_loss": 3.47561931447618, + "tokens_seen": 1713307648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890909090909091, + "loss": 1.2454, + "theoretical_loss": 3.475607889829428, + "tokens_seen": 1713373184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890707070707071, + "loss": 1.5084, + "theoretical_loss": 3.475596465742009, + "tokens_seen": 1713438720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890505050505051, + "loss": 1.5201, + "theoretical_loss": 3.475585042213874, + "tokens_seen": 1713504256 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489030303030303, + "loss": 1.6822, + "theoretical_loss": 3.4755736192449733, + "tokens_seen": 1713569792 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489010101010101, + "loss": 1.4501, + "theoretical_loss": 3.4755621968352592, + "tokens_seen": 1713635328 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488989898989899, + "loss": 1.4203, + "theoretical_loss": 3.475550774984682, + "tokens_seen": 1713700864 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889696969696969, + "loss": 1.4855, + "theoretical_loss": 3.4755393536931942, + "tokens_seen": 1713766400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889494949494949, + "loss": 1.3609, + "theoretical_loss": 3.4755279329607465, + "tokens_seen": 1713831936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889292929292929, + "loss": 1.5888, + "theoretical_loss": 3.4755165127872893, + "tokens_seen": 1713897472 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488909090909091, + "loss": 1.5043, + "theoretical_loss": 3.4755050931727753, + "tokens_seen": 1713963008 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.5464931726455688, + "objective/train/avg_token_score": 0.0072470419108867645, + "objective/train/docs_used": 971016, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7423732280731201, + "objective/train/original_loss": 3.5915262699127197, + "objective/train/theoretical_loss": 3.4754936741171543, + "objective/train/tokens_used": 72887776, + "objective/train/unlikelihood_loss": 0.07072310149669647, + "theoretical_loss": 3.4754936741171543, + "tokens_seen": 1714028544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888888888888889, + "loss": 1.4365, + "theoretical_loss": 3.4754936741171543, + "tokens_seen": 1714028544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888686868686869, + "loss": 1.434, + "theoretical_loss": 3.475482255620379, + "tokens_seen": 1714094080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888484848484849, + "loss": 1.1613, + "theoretical_loss": 3.4754708376823995, + "tokens_seen": 1714159616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888282828282829, + "loss": 1.2823, + "theoretical_loss": 3.475459420303168, + "tokens_seen": 1714225152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888080808080808, + "loss": 1.2084, + "theoretical_loss": 3.4754480034826356, + "tokens_seen": 1714290688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887878787878788, + "loss": 1.1509, + "theoretical_loss": 3.475436587220753, + "tokens_seen": 1714356224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887676767676768, + "loss": 1.4293, + "theoretical_loss": 3.4754251715174727, + "tokens_seen": 1714421760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887474747474748, + "loss": 1.4948, + "theoretical_loss": 3.4754137563727445, + "tokens_seen": 1714487296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887272727272727, + "loss": 1.3427, + "theoretical_loss": 3.4754023417865207, + "tokens_seen": 1714552832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887070707070707, + "loss": 1.393, + "theoretical_loss": 3.4753909277587525, + "tokens_seen": 1714618368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886868686868686, + "loss": 1.4489, + "theoretical_loss": 3.475379514289391, + "tokens_seen": 1714683904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886666666666667, + "loss": 1.3675, + "theoretical_loss": 3.475368101378388, + "tokens_seen": 1714749440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886464646464647, + "loss": 1.2169, + "theoretical_loss": 3.475356689025694, + "tokens_seen": 1714814976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886262626262627, + "loss": 1.5594, + "theoretical_loss": 3.4753452772312614, + "tokens_seen": 1714880512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886060606060606, + "loss": 1.3374, + "theoretical_loss": 3.475333865995041, + "tokens_seen": 1714946048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885858585858586, + "loss": 1.3952, + "theoretical_loss": 3.4753224553169835, + "tokens_seen": 1715011584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885656565656566, + "loss": 1.4714, + "theoretical_loss": 3.4753110451970417, + "tokens_seen": 1715077120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885454545454546, + "loss": 1.2954, + "theoretical_loss": 3.475299635635166, + "tokens_seen": 1715142656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885252525252525, + "loss": 1.5126, + "theoretical_loss": 3.475288226631308, + "tokens_seen": 1715208192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885050505050505, + "loss": 1.307, + "theoretical_loss": 3.475276818185419, + "tokens_seen": 1715273728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884848484848484, + "loss": 1.3257, + "theoretical_loss": 3.475265410297451, + "tokens_seen": 1715339264 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884646464646465, + "loss": 1.3537, + "theoretical_loss": 3.4752540029673544, + "tokens_seen": 1715404800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884444444444445, + "loss": 1.3866, + "theoretical_loss": 3.4752425961950815, + "tokens_seen": 1715470336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884242424242425, + "loss": 1.4019, + "theoretical_loss": 3.4752311899805832, + "tokens_seen": 1715535872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884040404040404, + "loss": 1.1464, + "theoretical_loss": 3.475219784323811, + "tokens_seen": 1715601408 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.4002932608127594, + "objective/train/avg_token_score": 0.017858585342764854, + "objective/train/docs_used": 972281, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3489974737167358, + "objective/train/original_loss": 4.905088424682617, + "objective/train/theoretical_loss": 3.475208379224716, + "objective/train/tokens_used": 74526176, + "objective/train/unlikelihood_loss": 0.05417528748512268, + "theoretical_loss": 3.475208379224716, + "tokens_seen": 1715666944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883838383838384, + "loss": 1.526, + "theoretical_loss": 3.475208379224716, + "tokens_seen": 1715666944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883636363636364, + "loss": 1.2685, + "theoretical_loss": 3.475196974683251, + "tokens_seen": 1715732480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883434343434344, + "loss": 1.4254, + "theoretical_loss": 3.4751855706993657, + "tokens_seen": 1715798016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883232323232323, + "loss": 1.3064, + "theoretical_loss": 3.4751741672730123, + "tokens_seen": 1715863552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883030303030303, + "loss": 1.3617, + "theoretical_loss": 3.4751627644041427, + "tokens_seen": 1715929088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882828282828283, + "loss": 1.4178, + "theoretical_loss": 3.4751513620927073, + "tokens_seen": 1715994624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882626262626263, + "loss": 1.6226, + "theoretical_loss": 3.475139960338659, + "tokens_seen": 1716060160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882424242424243, + "loss": 1.4401, + "theoretical_loss": 3.4751285591419476, + "tokens_seen": 1716125696 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048822222222222225, + "loss": 1.3443, + "theoretical_loss": 3.475117158502526, + "tokens_seen": 1716191232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882020202020202, + "loss": 1.4155, + "theoretical_loss": 3.4751057584203444, + "tokens_seen": 1716256768 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048818181818181816, + "loss": 1.2397, + "theoretical_loss": 3.4750943588953556, + "tokens_seen": 1716322304 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048816161616161617, + "loss": 1.2896, + "theoretical_loss": 3.47508295992751, + "tokens_seen": 1716387840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881414141414142, + "loss": 1.4234, + "theoretical_loss": 3.47507156151676, + "tokens_seen": 1716453376 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048812121212121214, + "loss": 1.6226, + "theoretical_loss": 3.4750601636630565, + "tokens_seen": 1716518912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881010101010101, + "loss": 1.3232, + "theoretical_loss": 3.475048766366351, + "tokens_seen": 1716584448 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048808080808080805, + "loss": 1.2095, + "theoretical_loss": 3.4750373696265955, + "tokens_seen": 1716649984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880606060606061, + "loss": 1.3368, + "theoretical_loss": 3.4750259734437408, + "tokens_seen": 1716715520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880404040404041, + "loss": 1.5763, + "theoretical_loss": 3.4750145778177393, + "tokens_seen": 1716781056 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048802020202020203, + "loss": 1.537, + "theoretical_loss": 3.4750031827485417, + "tokens_seen": 1716846592 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488, + "loss": 1.4221, + "theoretical_loss": 3.4749917882360997, + "tokens_seen": 1716912128 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048797979797979795, + "loss": 1.4776, + "theoretical_loss": 3.4749803942803656, + "tokens_seen": 1716977664 + }, + { + "epoch": 0.03, + "learning_rate": 0.000487959595959596, + "loss": 1.6911, + "theoretical_loss": 3.4749690008812903, + "tokens_seen": 1717043200 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048793939393939397, + "loss": 1.6163, + "theoretical_loss": 3.4749576080388254, + "tokens_seen": 1717108736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879191919191919, + "loss": 1.5933, + "theoretical_loss": 3.4749462157529227, + "tokens_seen": 1717174272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878989898989899, + "loss": 1.4458, + "theoretical_loss": 3.4749348240235336, + "tokens_seen": 1717239808 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.34756842255592346, + "objective/train/avg_token_score": 0.022121969610452652, + "objective/train/docs_used": 972817, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2492351531982422, + "objective/train/original_loss": 4.464137554168701, + "objective/train/theoretical_loss": 3.47492343285061, + "objective/train/tokens_used": 76164576, + "objective/train/unlikelihood_loss": 0.06865523010492325, + "theoretical_loss": 3.47492343285061, + "tokens_seen": 1717305344 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048787878787878784, + "loss": 1.4663, + "theoretical_loss": 3.47492343285061, + "tokens_seen": 1717305344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878585858585859, + "loss": 1.6475, + "theoretical_loss": 3.474912042234102, + "tokens_seen": 1717370880 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048783838383838386, + "loss": 1.3615, + "theoretical_loss": 3.4749006521739636, + "tokens_seen": 1717436416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878181818181818, + "loss": 1.3724, + "theoretical_loss": 3.4748892626701444, + "tokens_seen": 1717501952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877979797979798, + "loss": 1.4726, + "theoretical_loss": 3.4748778737225976, + "tokens_seen": 1717567488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877777777777778, + "loss": 1.5526, + "theoretical_loss": 3.4748664853312734, + "tokens_seen": 1717633024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877575757575758, + "loss": 1.3223, + "theoretical_loss": 3.474855097496124, + "tokens_seen": 1717698560 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048773737373737375, + "loss": 1.6783, + "theoretical_loss": 3.474843710217101, + "tokens_seen": 1717764096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877171717171717, + "loss": 1.3255, + "theoretical_loss": 3.474832323494156, + "tokens_seen": 1717829632 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048769696969696967, + "loss": 1.3216, + "theoretical_loss": 3.474820937327241, + "tokens_seen": 1717895168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876767676767677, + "loss": 1.3008, + "theoretical_loss": 3.474809551716307, + "tokens_seen": 1717960704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876565656565657, + "loss": 1.1347, + "theoretical_loss": 3.474798166661306, + "tokens_seen": 1718026240 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048763636363636365, + "loss": 1.4412, + "theoretical_loss": 3.4747867821621896, + "tokens_seen": 1718091776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876161616161616, + "loss": 1.1156, + "theoretical_loss": 3.4747753982189096, + "tokens_seen": 1718157312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875959595959596, + "loss": 1.4876, + "theoretical_loss": 3.474764014831417, + "tokens_seen": 1718222848 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048757575757575757, + "loss": 1.4029, + "theoretical_loss": 3.4747526319996647, + "tokens_seen": 1718288384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875555555555556, + "loss": 1.3172, + "theoretical_loss": 3.474741249723603, + "tokens_seen": 1718353920 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048753535353535354, + "loss": 1.2698, + "theoretical_loss": 3.4747298680031844, + "tokens_seen": 1718419456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875151515151515, + "loss": 1.5606, + "theoretical_loss": 3.4747184868383605, + "tokens_seen": 1718484992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874949494949495, + "loss": 1.4975, + "theoretical_loss": 3.474707106229083, + "tokens_seen": 1718550528 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048747474747474746, + "loss": 1.5241, + "theoretical_loss": 3.4746957261753035, + "tokens_seen": 1718616064 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874545454545455, + "loss": 1.5173, + "theoretical_loss": 3.4746843466769737, + "tokens_seen": 1718681600 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048743434343434343, + "loss": 1.5222, + "theoretical_loss": 3.4746729677340453, + "tokens_seen": 1718747136 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048741414141414144, + "loss": 1.4428, + "theoretical_loss": 3.47466158934647, + "tokens_seen": 1718812672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004873939393939394, + "loss": 1.2955, + "theoretical_loss": 3.4746502115141995, + "tokens_seen": 1718878208 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.3179985284805298, + "objective/train/avg_token_score": 0.02353089302778244, + "objective/train/docs_used": 973424, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0046734809875488, + "objective/train/original_loss": 4.215363502502441, + "objective/train/theoretical_loss": 3.4746388342371857, + "objective/train/tokens_used": 77802976, + "objective/train/unlikelihood_loss": 0.07478584349155426, + "theoretical_loss": 3.4746388342371857, + "tokens_seen": 1718943744 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048737373737373736, + "loss": 1.2882, + "theoretical_loss": 3.4746388342371857, + "tokens_seen": 1718943744 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048735353535353537, + "loss": 1.5482, + "theoretical_loss": 3.47462745751538, + "tokens_seen": 1719009280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873333333333333, + "loss": 1.1838, + "theoretical_loss": 3.474616081348735, + "tokens_seen": 1719074816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048731313131313134, + "loss": 1.438, + "theoretical_loss": 3.4746047057372014, + "tokens_seen": 1719140352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872929292929293, + "loss": 1.4355, + "theoretical_loss": 3.4745933306807313, + "tokens_seen": 1719205888 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048727272727272725, + "loss": 1.2308, + "theoretical_loss": 3.474581956179277, + "tokens_seen": 1719271424 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048725252525252526, + "loss": 1.5441, + "theoretical_loss": 3.474570582232789, + "tokens_seen": 1719336960 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048723232323232327, + "loss": 1.4567, + "theoretical_loss": 3.4745592088412205, + "tokens_seen": 1719402496 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048721212121212123, + "loss": 1.5781, + "theoretical_loss": 3.4745478360045228, + "tokens_seen": 1719468032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871919191919192, + "loss": 1.4201, + "theoretical_loss": 3.4745364637226466, + "tokens_seen": 1719533568 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048717171717171714, + "loss": 1.5531, + "theoretical_loss": 3.4745250919955453, + "tokens_seen": 1719599104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048715151515151515, + "loss": 1.0908, + "theoretical_loss": 3.47451372082317, + "tokens_seen": 1719664640 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048713131313131316, + "loss": 1.6216, + "theoretical_loss": 3.474502350205472, + "tokens_seen": 1719730176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871111111111111, + "loss": 1.2598, + "theoretical_loss": 3.474490980142404, + "tokens_seen": 1719795712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870909090909091, + "loss": 1.3527, + "theoretical_loss": 3.4744796106339173, + "tokens_seen": 1719861248 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048707070707070704, + "loss": 1.3626, + "theoretical_loss": 3.474468241679964, + "tokens_seen": 1719926784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870505050505051, + "loss": 1.4276, + "theoretical_loss": 3.474456873280496, + "tokens_seen": 1719992320 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048703030303030306, + "loss": 1.5703, + "theoretical_loss": 3.4744455054354644, + "tokens_seen": 1720057856 + }, + { + "epoch": 0.04, + "learning_rate": 0.000487010101010101, + "loss": 1.4335, + "theoretical_loss": 3.474434138144822, + "tokens_seen": 1720123392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048698989898989897, + "loss": 1.4018, + "theoretical_loss": 3.47442277140852, + "tokens_seen": 1720188928 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486969696969697, + "loss": 1.4003, + "theoretical_loss": 3.47441140522651, + "tokens_seen": 1720254464 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486949494949495, + "loss": 1.5975, + "theoretical_loss": 3.474400039598745, + "tokens_seen": 1720320000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048692929292929295, + "loss": 1.3209, + "theoretical_loss": 3.4743886745251755, + "tokens_seen": 1720385536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869090909090909, + "loss": 1.5469, + "theoretical_loss": 3.4743773100057544, + "tokens_seen": 1720451072 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048688888888888886, + "loss": 1.3673, + "theoretical_loss": 3.4743659460404333, + "tokens_seen": 1720516608 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.5211998820304871, + "objective/train/avg_token_score": 0.0027845173608511686, + "objective/train/docs_used": 974650, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7277867794036865, + "objective/train/original_loss": 3.4647514820098877, + "objective/train/theoretical_loss": 3.4743545826291635, + "objective/train/tokens_used": 79441376, + "objective/train/unlikelihood_loss": 0.10759128630161285, + "theoretical_loss": 3.4743545826291635, + "tokens_seen": 1720582144 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048686868686868693, + "loss": 1.5171, + "theoretical_loss": 3.4743545826291635, + "tokens_seen": 1720582144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868484848484849, + "loss": 1.3346, + "theoretical_loss": 3.4743432197718978, + "tokens_seen": 1720647680 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048682828282828284, + "loss": 1.2947, + "theoretical_loss": 3.4743318574685875, + "tokens_seen": 1720713216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868080808080808, + "loss": 1.3649, + "theoretical_loss": 3.4743204957191844, + "tokens_seen": 1720778752 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048678787878787876, + "loss": 1.3602, + "theoretical_loss": 3.474309134523641, + "tokens_seen": 1720844288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867676767676768, + "loss": 1.3379, + "theoretical_loss": 3.4742977738819087, + "tokens_seen": 1720909824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867474747474748, + "loss": 1.5094, + "theoretical_loss": 3.4742864137939398, + "tokens_seen": 1720975360 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048672727272727274, + "loss": 1.4478, + "theoretical_loss": 3.474275054259685, + "tokens_seen": 1721040896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867070707070707, + "loss": 1.4563, + "theoretical_loss": 3.4742636952790984, + "tokens_seen": 1721106432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866868686868687, + "loss": 1.5572, + "theoretical_loss": 3.47425233685213, + "tokens_seen": 1721171968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866666666666667, + "loss": 1.3569, + "theoretical_loss": 3.474240978978733, + "tokens_seen": 1721237504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048664646464646467, + "loss": 1.2837, + "theoretical_loss": 3.474229621658859, + "tokens_seen": 1721303040 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048662626262626263, + "loss": 1.4573, + "theoretical_loss": 3.474218264892459, + "tokens_seen": 1721368576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866060606060606, + "loss": 1.4406, + "theoretical_loss": 3.4742069086794864, + "tokens_seen": 1721434112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865858585858586, + "loss": 1.2444, + "theoretical_loss": 3.474195553019892, + "tokens_seen": 1721499648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865656565656566, + "loss": 1.3948, + "theoretical_loss": 3.4741841979136288, + "tokens_seen": 1721565184 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048654545454545456, + "loss": 1.4, + "theoretical_loss": 3.4741728433606482, + "tokens_seen": 1721630720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865252525252525, + "loss": 1.5166, + "theoretical_loss": 3.4741614893609016, + "tokens_seen": 1721696256 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048650505050505053, + "loss": 1.5006, + "theoretical_loss": 3.4741501359143423, + "tokens_seen": 1721761792 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864848484848485, + "loss": 1.2737, + "theoretical_loss": 3.4741387830209214, + "tokens_seen": 1721827328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864646464646465, + "loss": 1.3364, + "theoretical_loss": 3.474127430680591, + "tokens_seen": 1721892864 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048644444444444446, + "loss": 1.4177, + "theoretical_loss": 3.474116078893303, + "tokens_seen": 1721958400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864242424242424, + "loss": 1.3975, + "theoretical_loss": 3.47410472765901, + "tokens_seen": 1722023936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864040404040404, + "loss": 1.3059, + "theoretical_loss": 3.474093376977664, + "tokens_seen": 1722089472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863838383838384, + "loss": 1.4123, + "theoretical_loss": 3.474082026849216, + "tokens_seen": 1722155008 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.3375488817691803, + "objective/train/avg_token_score": 0.013777061365544796, + "objective/train/docs_used": 975130, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2517693042755127, + "objective/train/original_loss": 5.1249237060546875, + "objective/train/theoretical_loss": 3.4740706772736187, + "objective/train/tokens_used": 81079776, + "objective/train/unlikelihood_loss": 0.040009669959545135, + "theoretical_loss": 3.4740706772736187, + "tokens_seen": 1722220544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863636363636364, + "loss": 1.5127, + "theoretical_loss": 3.4740706772736187, + "tokens_seen": 1722220544 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048634343434343435, + "loss": 1.2964, + "theoretical_loss": 3.4740593282508243, + "tokens_seen": 1722286080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048632323232323236, + "loss": 1.4136, + "theoretical_loss": 3.474047979780785, + "tokens_seen": 1722351616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863030303030303, + "loss": 1.4117, + "theoretical_loss": 3.474036631863452, + "tokens_seen": 1722417152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862828282828283, + "loss": 1.3667, + "theoretical_loss": 3.4740252844987785, + "tokens_seen": 1722482688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862626262626263, + "loss": 1.2363, + "theoretical_loss": 3.474013937686715, + "tokens_seen": 1722548224 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048624242424242424, + "loss": 1.4347, + "theoretical_loss": 3.4740025914272152, + "tokens_seen": 1722613760 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048622222222222225, + "loss": 1.3785, + "theoretical_loss": 3.4739912457202307, + "tokens_seen": 1722679296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862020202020202, + "loss": 1.4418, + "theoretical_loss": 3.4739799005657126, + "tokens_seen": 1722744832 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048618181818181817, + "loss": 1.4983, + "theoretical_loss": 3.473968555963614, + "tokens_seen": 1722810368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861616161616162, + "loss": 1.4721, + "theoretical_loss": 3.473957211913887, + "tokens_seen": 1722875904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861414141414142, + "loss": 1.4795, + "theoretical_loss": 3.4739458684164832, + "tokens_seen": 1722941440 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048612121212121215, + "loss": 1.3607, + "theoretical_loss": 3.473934525471355, + "tokens_seen": 1723006976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861010101010101, + "loss": 1.4223, + "theoretical_loss": 3.4739231830784543, + "tokens_seen": 1723072512 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048608080808080806, + "loss": 1.5619, + "theoretical_loss": 3.473911841237733, + "tokens_seen": 1723138048 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048606060606060607, + "loss": 1.4897, + "theoretical_loss": 3.473900499949144, + "tokens_seen": 1723203584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860404040404041, + "loss": 1.3993, + "theoretical_loss": 3.4738891592126393, + "tokens_seen": 1723269120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048602020202020204, + "loss": 1.408, + "theoretical_loss": 3.4738778190281705, + "tokens_seen": 1723334656 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486, + "loss": 1.3879, + "theoretical_loss": 3.4738664793956895, + "tokens_seen": 1723400192 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048597979797979795, + "loss": 1.4359, + "theoretical_loss": 3.473855140315149, + "tokens_seen": 1723465728 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048595959595959596, + "loss": 1.4494, + "theoretical_loss": 3.473843801786501, + "tokens_seen": 1723531264 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485939393939394, + "loss": 1.3349, + "theoretical_loss": 3.473832463809698, + "tokens_seen": 1723596800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048591919191919193, + "loss": 1.4685, + "theoretical_loss": 3.4738211263846916, + "tokens_seen": 1723662336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858989898989899, + "loss": 1.3586, + "theoretical_loss": 3.473809789511434, + "tokens_seen": 1723727872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048587878787878784, + "loss": 1.3079, + "theoretical_loss": 3.473798453189878, + "tokens_seen": 1723793408 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.4409824013710022, + "objective/train/avg_token_score": 0.026003144681453705, + "objective/train/docs_used": 976352, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4241589307785034, + "objective/train/original_loss": 4.103183746337891, + "objective/train/theoretical_loss": 3.4737871174199757, + "objective/train/tokens_used": 82718176, + "objective/train/unlikelihood_loss": 0.09437726438045502, + "theoretical_loss": 3.4737871174199757, + "tokens_seen": 1723858944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858585858585859, + "loss": 1.4386, + "theoretical_loss": 3.4737871174199757, + "tokens_seen": 1723858944 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048583838383838387, + "loss": 1.3626, + "theoretical_loss": 3.4737757822016784, + "tokens_seen": 1723924480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858181818181818, + "loss": 1.4696, + "theoretical_loss": 3.473764447534939, + "tokens_seen": 1723990016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857979797979798, + "loss": 1.3754, + "theoretical_loss": 3.4737531134197095, + "tokens_seen": 1724055552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048577777777777774, + "loss": 1.4346, + "theoretical_loss": 3.473741779855942, + "tokens_seen": 1724121088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857575757575758, + "loss": 1.4929, + "theoretical_loss": 3.4737304468435894, + "tokens_seen": 1724186624 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048573737373737376, + "loss": 1.4366, + "theoretical_loss": 3.473719114382603, + "tokens_seen": 1724252160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857171717171717, + "loss": 1.2766, + "theoretical_loss": 3.4737077824729354, + "tokens_seen": 1724317696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856969696969697, + "loss": 1.3714, + "theoretical_loss": 3.4736964511145385, + "tokens_seen": 1724383232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856767676767677, + "loss": 1.2763, + "theoretical_loss": 3.473685120307365, + "tokens_seen": 1724448768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856565656565657, + "loss": 1.4836, + "theoretical_loss": 3.4736737900513672, + "tokens_seen": 1724514304 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048563636363636365, + "loss": 1.4698, + "theoretical_loss": 3.473662460346497, + "tokens_seen": 1724579840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856161616161616, + "loss": 1.4449, + "theoretical_loss": 3.4736511311927067, + "tokens_seen": 1724645376 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048559595959595957, + "loss": 1.3052, + "theoretical_loss": 3.4736398025899486, + "tokens_seen": 1724710912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855757575757576, + "loss": 1.565, + "theoretical_loss": 3.4736284745381756, + "tokens_seen": 1724776448 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855555555555556, + "loss": 1.3919, + "theoretical_loss": 3.4736171470373387, + "tokens_seen": 1724841984 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048553535353535355, + "loss": 1.5023, + "theoretical_loss": 3.473605820087391, + "tokens_seen": 1724907520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855151515151515, + "loss": 1.2998, + "theoretical_loss": 3.4735944936882848, + "tokens_seen": 1724973056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854949494949495, + "loss": 1.6207, + "theoretical_loss": 3.473583167839972, + "tokens_seen": 1725038592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048547474747474747, + "loss": 1.2696, + "theoretical_loss": 3.473571842542405, + "tokens_seen": 1725104128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854545454545455, + "loss": 1.6765, + "theoretical_loss": 3.473560517795536, + "tokens_seen": 1725169664 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048543434343434344, + "loss": 1.5439, + "theoretical_loss": 3.473549193599318, + "tokens_seen": 1725235200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854141414141414, + "loss": 1.5656, + "theoretical_loss": 3.4735378699537027, + "tokens_seen": 1725300736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853939393939394, + "loss": 1.2932, + "theoretical_loss": 3.4735265468586425, + "tokens_seen": 1725366272 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048537373737373736, + "loss": 1.4775, + "theoretical_loss": 3.4735152243140894, + "tokens_seen": 1725431808 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.4415322542190552, + "objective/train/avg_token_score": 0.00970387365669012, + "objective/train/docs_used": 977030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4506078958511353, + "objective/train/original_loss": 4.433774948120117, + "objective/train/theoretical_loss": 3.4735039023199965, + "objective/train/tokens_used": 84356576, + "objective/train/unlikelihood_loss": 0.02620289847254753, + "theoretical_loss": 3.4735039023199965, + "tokens_seen": 1725497344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853535353535354, + "loss": 1.7168, + "theoretical_loss": 3.4735039023199965, + "tokens_seen": 1725497344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048533333333333333, + "loss": 1.2021, + "theoretical_loss": 3.4734925808763153, + "tokens_seen": 1725562880 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048531313131313134, + "loss": 1.4966, + "theoretical_loss": 3.473481259982999, + "tokens_seen": 1725628416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852929292929293, + "loss": 1.4124, + "theoretical_loss": 3.4734699396399993, + "tokens_seen": 1725693952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852727272727273, + "loss": 1.3887, + "theoretical_loss": 3.4734586198472686, + "tokens_seen": 1725759488 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048525252525252527, + "loss": 1.1216, + "theoretical_loss": 3.4734473006047595, + "tokens_seen": 1725825024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852323232323232, + "loss": 1.3029, + "theoretical_loss": 3.4734359819124245, + "tokens_seen": 1725890560 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048521212121212123, + "loss": 1.3902, + "theoretical_loss": 3.4734246637702153, + "tokens_seen": 1725956096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851919191919192, + "loss": 1.34, + "theoretical_loss": 3.4734133461780847, + "tokens_seen": 1726021632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851717171717172, + "loss": 1.4358, + "theoretical_loss": 3.4734020291359853, + "tokens_seen": 1726087168 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048515151515151516, + "loss": 1.4642, + "theoretical_loss": 3.473390712643869, + "tokens_seen": 1726152704 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048513131313131317, + "loss": 1.3417, + "theoretical_loss": 3.473379396701689, + "tokens_seen": 1726218240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851111111111111, + "loss": 1.1703, + "theoretical_loss": 3.473368081309397, + "tokens_seen": 1726283776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850909090909091, + "loss": 1.6721, + "theoretical_loss": 3.473356766466945, + "tokens_seen": 1726349312 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850707070707071, + "loss": 1.3141, + "theoretical_loss": 3.4733454521742866, + "tokens_seen": 1726414848 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048505050505050505, + "loss": 1.2502, + "theoretical_loss": 3.4733341384313734, + "tokens_seen": 1726480384 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048503030303030306, + "loss": 1.2986, + "theoretical_loss": 3.4733228252381574, + "tokens_seen": 1726545920 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485010101010101, + "loss": 1.3311, + "theoretical_loss": 3.4733115125945924, + "tokens_seen": 1726611456 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484989898989899, + "loss": 1.5198, + "theoretical_loss": 3.47330020050063, + "tokens_seen": 1726676992 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484969696969697, + "loss": 1.2449, + "theoretical_loss": 3.4732888889562226, + "tokens_seen": 1726742528 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484949494949495, + "loss": 1.345, + "theoretical_loss": 3.4732775779613227, + "tokens_seen": 1726808064 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048492929292929296, + "loss": 1.3956, + "theoretical_loss": 3.473266267515883, + "tokens_seen": 1726873600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849090909090909, + "loss": 1.3801, + "theoretical_loss": 3.4732549576198553, + "tokens_seen": 1726939136 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048488888888888887, + "loss": 1.3834, + "theoretical_loss": 3.4732436482731925, + "tokens_seen": 1727004672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848686868686869, + "loss": 1.4937, + "theoretical_loss": 3.4732323394758478, + "tokens_seen": 1727070208 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.39345064759254456, + "objective/train/avg_token_score": 0.009139729663729668, + "objective/train/docs_used": 978106, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4235575199127197, + "objective/train/original_loss": 4.360210418701172, + "objective/train/theoretical_loss": 3.473221031227773, + "objective/train/tokens_used": 85994976, + "objective/train/unlikelihood_loss": 0.08843415230512619, + "theoretical_loss": 3.473221031227773, + "tokens_seen": 1727135744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848484848484849, + "loss": 1.5847, + "theoretical_loss": 3.473221031227773, + "tokens_seen": 1727135744 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048482828282828285, + "loss": 1.4794, + "theoretical_loss": 3.47320972352892, + "tokens_seen": 1727201280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848080808080808, + "loss": 1.4167, + "theoretical_loss": 3.473198416379242, + "tokens_seen": 1727266816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048478787878787876, + "loss": 1.2351, + "theoretical_loss": 3.4731871097786913, + "tokens_seen": 1727332352 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048476767676767683, + "loss": 1.3563, + "theoretical_loss": 3.4731758037272202, + "tokens_seen": 1727397888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847474747474748, + "loss": 1.291, + "theoretical_loss": 3.473164498224782, + "tokens_seen": 1727463424 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048472727272727274, + "loss": 1.4047, + "theoretical_loss": 3.4731531932713287, + "tokens_seen": 1727528960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847070707070707, + "loss": 1.3456, + "theoretical_loss": 3.473141888866812, + "tokens_seen": 1727594496 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048468686868686865, + "loss": 1.3892, + "theoretical_loss": 3.4731305850111855, + "tokens_seen": 1727660032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846666666666667, + "loss": 1.3859, + "theoretical_loss": 3.473119281704402, + "tokens_seen": 1727725568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846464646464647, + "loss": 1.1975, + "theoretical_loss": 3.473107978946413, + "tokens_seen": 1727791104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048462626262626263, + "loss": 1.3792, + "theoretical_loss": 3.4730966767371716, + "tokens_seen": 1727856640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846060606060606, + "loss": 1.3253, + "theoretical_loss": 3.4730853750766304, + "tokens_seen": 1727922176 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048458585858585855, + "loss": 1.5187, + "theoretical_loss": 3.4730740739647414, + "tokens_seen": 1727987712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845656565656566, + "loss": 1.4738, + "theoretical_loss": 3.4730627734014576, + "tokens_seen": 1728053248 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048454545454545457, + "loss": 1.1208, + "theoretical_loss": 3.4730514733867315, + "tokens_seen": 1728118784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845252525252525, + "loss": 1.3628, + "theoretical_loss": 3.4730401739205163, + "tokens_seen": 1728184320 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845050505050505, + "loss": 1.4873, + "theoretical_loss": 3.4730288750027634, + "tokens_seen": 1728249856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844848484848485, + "loss": 1.3379, + "theoretical_loss": 3.473017576633426, + "tokens_seen": 1728315392 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844646464646465, + "loss": 1.2096, + "theoretical_loss": 3.4730062788124565, + "tokens_seen": 1728380928 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048444444444444446, + "loss": 1.2331, + "theoretical_loss": 3.472994981539808, + "tokens_seen": 1728446464 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844242424242424, + "loss": 1.5172, + "theoretical_loss": 3.4729836848154325, + "tokens_seen": 1728512000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844040404040404, + "loss": 1.4394, + "theoretical_loss": 3.472972388639283, + "tokens_seen": 1728577536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843838383838384, + "loss": 1.4188, + "theoretical_loss": 3.4729610930113117, + "tokens_seen": 1728643072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843636363636364, + "loss": 1.305, + "theoretical_loss": 3.472949797931472, + "tokens_seen": 1728708608 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.3660801649093628, + "objective/train/avg_token_score": 0.007394853513687849, + "objective/train/docs_used": 978835, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.256290316581726, + "objective/train/original_loss": 4.4458208084106445, + "objective/train/theoretical_loss": 3.472938503399715, + "objective/train/tokens_used": 87633376, + "objective/train/unlikelihood_loss": 0.06266473978757858, + "theoretical_loss": 3.472938503399715, + "tokens_seen": 1728774144 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048434343434343435, + "loss": 1.5173, + "theoretical_loss": 3.472938503399715, + "tokens_seen": 1728774144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843232323232323, + "loss": 1.6362, + "theoretical_loss": 3.472927209415995, + "tokens_seen": 1728839680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843030303030303, + "loss": 1.2885, + "theoretical_loss": 3.4729159159802636, + "tokens_seen": 1728905216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842828282828283, + "loss": 1.5108, + "theoretical_loss": 3.4729046230924743, + "tokens_seen": 1728970752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842626262626263, + "loss": 1.5734, + "theoretical_loss": 3.472893330752579, + "tokens_seen": 1729036288 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048424242424242425, + "loss": 1.2754, + "theoretical_loss": 3.4728820389605306, + "tokens_seen": 1729101824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842222222222222, + "loss": 1.3543, + "theoretical_loss": 3.4728707477162817, + "tokens_seen": 1729167360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842020202020202, + "loss": 1.4882, + "theoretical_loss": 3.472859457019785, + "tokens_seen": 1729232896 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048418181818181817, + "loss": 1.288, + "theoretical_loss": 3.4728481668709934, + "tokens_seen": 1729298432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841616161616162, + "loss": 1.1096, + "theoretical_loss": 3.4728368772698586, + "tokens_seen": 1729363968 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048414141414141414, + "loss": 1.3439, + "theoretical_loss": 3.4728255882163346, + "tokens_seen": 1729429504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048412121212121215, + "loss": 1.3121, + "theoretical_loss": 3.4728142997103735, + "tokens_seen": 1729495040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841010101010101, + "loss": 1.3209, + "theoretical_loss": 3.472803011751928, + "tokens_seen": 1729560576 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048408080808080807, + "loss": 1.3692, + "theoretical_loss": 3.472791724340951, + "tokens_seen": 1729626112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840606060606061, + "loss": 1.5298, + "theoretical_loss": 3.472780437477395, + "tokens_seen": 1729691648 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048404040404040403, + "loss": 1.4671, + "theoretical_loss": 3.472769151161213, + "tokens_seen": 1729757184 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048402020202020204, + "loss": 1.5067, + "theoretical_loss": 3.4727578653923565, + "tokens_seen": 1729822720 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484, + "loss": 1.5815, + "theoretical_loss": 3.4727465801707798, + "tokens_seen": 1729888256 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048397979797979796, + "loss": 1.5065, + "theoretical_loss": 3.472735295496435, + "tokens_seen": 1729953792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048395959595959597, + "loss": 1.3621, + "theoretical_loss": 3.4727240113692748, + "tokens_seen": 1730019328 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483939393939394, + "loss": 1.2581, + "theoretical_loss": 3.472712727789252, + "tokens_seen": 1730084864 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048391919191919194, + "loss": 1.4585, + "theoretical_loss": 3.4727014447563196, + "tokens_seen": 1730150400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838989898989899, + "loss": 1.3508, + "theoretical_loss": 3.472690162270429, + "tokens_seen": 1730215936 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048387878787878785, + "loss": 1.4799, + "theoretical_loss": 3.472678880331535, + "tokens_seen": 1730281472 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048385858585858586, + "loss": 1.3777, + "theoretical_loss": 3.4726675989395894, + "tokens_seen": 1730347008 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.46120479702949524, + "objective/train/avg_token_score": 0.012367086485028267, + "objective/train/docs_used": 979527, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5714759826660156, + "objective/train/original_loss": 4.267512321472168, + "objective/train/theoretical_loss": 3.472656318094545, + "objective/train/tokens_used": 89271776, + "objective/train/unlikelihood_loss": 0.061104901134967804, + "theoretical_loss": 3.472656318094545, + "tokens_seen": 1730412544 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048383838383838387, + "loss": 1.3963, + "theoretical_loss": 3.472656318094545, + "tokens_seen": 1730412544 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048381818181818183, + "loss": 1.4758, + "theoretical_loss": 3.4726450377963545, + "tokens_seen": 1730478080 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837979797979798, + "loss": 1.2657, + "theoretical_loss": 3.47263375804497, + "tokens_seen": 1730543616 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048377777777777774, + "loss": 1.3777, + "theoretical_loss": 3.472622478840346, + "tokens_seen": 1730609152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837575757575758, + "loss": 1.4637, + "theoretical_loss": 3.472611200182434, + "tokens_seen": 1730674688 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048373737373737377, + "loss": 1.3547, + "theoretical_loss": 3.472599922071187, + "tokens_seen": 1730740224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837171717171717, + "loss": 1.4689, + "theoretical_loss": 3.472588644506558, + "tokens_seen": 1730805760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836969696969697, + "loss": 1.6301, + "theoretical_loss": 3.4725773674884994, + "tokens_seen": 1730871296 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048367676767676764, + "loss": 1.5216, + "theoretical_loss": 3.4725660910169647, + "tokens_seen": 1730936832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836565656565657, + "loss": 1.5067, + "theoretical_loss": 3.472554815091906, + "tokens_seen": 1731002368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048363636363636366, + "loss": 1.4398, + "theoretical_loss": 3.472543539713277, + "tokens_seen": 1731067904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836161616161616, + "loss": 1.2717, + "theoretical_loss": 3.4725322648810297, + "tokens_seen": 1731133440 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048359595959595957, + "loss": 1.3783, + "theoretical_loss": 3.4725209905951173, + "tokens_seen": 1731198976 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048357575757575764, + "loss": 1.5397, + "theoretical_loss": 3.472509716855493, + "tokens_seen": 1731264512 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835555555555556, + "loss": 1.4858, + "theoretical_loss": 3.4724984436621087, + "tokens_seen": 1731330048 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048353535353535355, + "loss": 1.3623, + "theoretical_loss": 3.4724871710149183, + "tokens_seen": 1731395584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835151515151515, + "loss": 1.3471, + "theoretical_loss": 3.4724758989138738, + "tokens_seen": 1731461120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048349494949494946, + "loss": 1.5417, + "theoretical_loss": 3.472464627358929, + "tokens_seen": 1731526656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048347474747474753, + "loss": 1.6138, + "theoretical_loss": 3.4724533563500355, + "tokens_seen": 1731592192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834545454545455, + "loss": 1.6054, + "theoretical_loss": 3.472442085887147, + "tokens_seen": 1731657728 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048343434343434344, + "loss": 1.6946, + "theoretical_loss": 3.472430815970217, + "tokens_seen": 1731723264 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834141414141414, + "loss": 1.4545, + "theoretical_loss": 3.4724195465991974, + "tokens_seen": 1731788800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833939393939394, + "loss": 1.4118, + "theoretical_loss": 3.4724082777740413, + "tokens_seen": 1731854336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833737373737374, + "loss": 1.2402, + "theoretical_loss": 3.472397009494702, + "tokens_seen": 1731919872 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833535353535354, + "loss": 1.5217, + "theoretical_loss": 3.4723857417611317, + "tokens_seen": 1731985408 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.37738269567489624, + "objective/train/avg_token_score": 0.007554531563073397, + "objective/train/docs_used": 980288, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1984022855758667, + "objective/train/original_loss": 4.563331127166748, + "objective/train/theoretical_loss": 3.4723744745732836, + "objective/train/tokens_used": 90910176, + "objective/train/unlikelihood_loss": 0.06496313959360123, + "theoretical_loss": 3.4723744745732836, + "tokens_seen": 1732050944 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048333333333333334, + "loss": 1.1324, + "theoretical_loss": 3.4723744745732836, + "tokens_seen": 1732050944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833131313131313, + "loss": 1.2802, + "theoretical_loss": 3.472363207931111, + "tokens_seen": 1732116480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832929292929293, + "loss": 1.3541, + "theoretical_loss": 3.4723519418345665, + "tokens_seen": 1732182016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832727272727273, + "loss": 1.5818, + "theoretical_loss": 3.4723406762836033, + "tokens_seen": 1732247552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048325252525252527, + "loss": 1.4111, + "theoretical_loss": 3.472329411278174, + "tokens_seen": 1732313088 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048323232323232323, + "loss": 1.4464, + "theoretical_loss": 3.472318146818232, + "tokens_seen": 1732378624 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048321212121212124, + "loss": 1.4059, + "theoretical_loss": 3.4723068829037294, + "tokens_seen": 1732444160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831919191919192, + "loss": 1.2797, + "theoretical_loss": 3.47229561953462, + "tokens_seen": 1732509696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831717171717172, + "loss": 1.3734, + "theoretical_loss": 3.4722843567108566, + "tokens_seen": 1732575232 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048315151515151516, + "loss": 1.3392, + "theoretical_loss": 3.4722730944323916, + "tokens_seen": 1732640768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831313131313131, + "loss": 1.4454, + "theoretical_loss": 3.4722618326991785, + "tokens_seen": 1732706304 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048311111111111113, + "loss": 1.578, + "theoretical_loss": 3.4722505715111707, + "tokens_seen": 1732771840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830909090909091, + "loss": 1.3528, + "theoretical_loss": 3.4722393108683205, + "tokens_seen": 1732837376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830707070707071, + "loss": 1.419, + "theoretical_loss": 3.472228050770581, + "tokens_seen": 1732902912 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048305050505050506, + "loss": 1.3326, + "theoretical_loss": 3.472216791217905, + "tokens_seen": 1732968448 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048303030303030307, + "loss": 1.6869, + "theoretical_loss": 3.4722055322102463, + "tokens_seen": 1733033984 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483010101010101, + "loss": 1.5042, + "theoretical_loss": 3.472194273747557, + "tokens_seen": 1733099520 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482989898989899, + "loss": 1.3897, + "theoretical_loss": 3.47218301582979, + "tokens_seen": 1733165056 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482969696969697, + "loss": 1.2143, + "theoretical_loss": 3.4721717584569003, + "tokens_seen": 1733230592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048294949494949495, + "loss": 1.5147, + "theoretical_loss": 3.472160501628838, + "tokens_seen": 1733296128 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048292929292929296, + "loss": 1.3647, + "theoretical_loss": 3.4721492453455585, + "tokens_seen": 1733361664 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829090909090909, + "loss": 1.1661, + "theoretical_loss": 3.4721379896070133, + "tokens_seen": 1733427200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828888888888889, + "loss": 1.5285, + "theoretical_loss": 3.4721267344131563, + "tokens_seen": 1733492736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828686868686869, + "loss": 1.4618, + "theoretical_loss": 3.4721154797639406, + "tokens_seen": 1733558272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828484848484849, + "loss": 1.5015, + "theoretical_loss": 3.4721042256593186, + "tokens_seen": 1733623808 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.5816226601600647, + "objective/train/avg_token_score": 0.014024200849235058, + "objective/train/docs_used": 981887, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.1395461559295654, + "objective/train/original_loss": 4.308298587799072, + "objective/train/theoretical_loss": 3.472092972099244, + "objective/train/tokens_used": 92548576, + "objective/train/unlikelihood_loss": 0.05324159935116768, + "theoretical_loss": 3.472092972099244, + "tokens_seen": 1733689344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048282828282828285, + "loss": 1.4546, + "theoretical_loss": 3.472092972099244, + "tokens_seen": 1733689344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828080808080808, + "loss": 1.1628, + "theoretical_loss": 3.4720817190836692, + "tokens_seen": 1733754880 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048278787878787877, + "loss": 1.2467, + "theoretical_loss": 3.472070466612548, + "tokens_seen": 1733820416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827676767676768, + "loss": 1.4641, + "theoretical_loss": 3.4720592146858333, + "tokens_seen": 1733885952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827474747474748, + "loss": 1.5125, + "theoretical_loss": 3.4720479633034778, + "tokens_seen": 1733951488 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048272727272727275, + "loss": 1.3324, + "theoretical_loss": 3.472036712465435, + "tokens_seen": 1734017024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827070707070707, + "loss": 1.3651, + "theoretical_loss": 3.4720254621716578, + "tokens_seen": 1734082560 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048268686868686866, + "loss": 1.4273, + "theoretical_loss": 3.472014212422099, + "tokens_seen": 1734148096 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048266666666666667, + "loss": 1.2076, + "theoretical_loss": 3.4720029632167124, + "tokens_seen": 1734213632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826464646464647, + "loss": 1.1455, + "theoretical_loss": 3.471991714555451, + "tokens_seen": 1734279168 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048262626262626264, + "loss": 1.5309, + "theoretical_loss": 3.471980466438267, + "tokens_seen": 1734344704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826060606060606, + "loss": 1.5117, + "theoretical_loss": 3.471969218865115, + "tokens_seen": 1734410240 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048258585858585855, + "loss": 1.5216, + "theoretical_loss": 3.4719579718359466, + "tokens_seen": 1734475776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825656565656566, + "loss": 1.52, + "theoretical_loss": 3.4719467253507164, + "tokens_seen": 1734541312 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825454545454546, + "loss": 1.5335, + "theoretical_loss": 3.471935479409376, + "tokens_seen": 1734606848 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048252525252525253, + "loss": 1.531, + "theoretical_loss": 3.4719242340118797, + "tokens_seen": 1734672384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825050505050505, + "loss": 1.3912, + "theoretical_loss": 3.471912989158181, + "tokens_seen": 1734737920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048248484848484845, + "loss": 1.5328, + "theoretical_loss": 3.471901744848231, + "tokens_seen": 1734803456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824646464646465, + "loss": 1.4363, + "theoretical_loss": 3.471890501081985, + "tokens_seen": 1734868992 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048244444444444447, + "loss": 1.6167, + "theoretical_loss": 3.4718792578593956, + "tokens_seen": 1734934528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824242424242424, + "loss": 1.3632, + "theoretical_loss": 3.4718680151804158, + "tokens_seen": 1735000064 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824040404040404, + "loss": 1.3851, + "theoretical_loss": 3.4718567730449985, + "tokens_seen": 1735065600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823838383838384, + "loss": 1.2801, + "theoretical_loss": 3.4718455314530976, + "tokens_seen": 1735131136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823636363636364, + "loss": 1.3273, + "theoretical_loss": 3.471834290404665, + "tokens_seen": 1735196672 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048234343434343436, + "loss": 1.5982, + "theoretical_loss": 3.4718230498996556, + "tokens_seen": 1735262208 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.25384896993637085, + "objective/train/avg_token_score": 0.01493894774466753, + "objective/train/docs_used": 982466, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8242332339286804, + "objective/train/original_loss": 4.889101505279541, + "objective/train/theoretical_loss": 3.4718118099380213, + "objective/train/tokens_used": 94186976, + "objective/train/unlikelihood_loss": 0.03157869726419449, + "theoretical_loss": 3.4718118099380213, + "tokens_seen": 1735327744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823232323232323, + "loss": 1.1415, + "theoretical_loss": 3.4718118099380213, + "tokens_seen": 1735327744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004823030303030303, + "loss": 1.321, + "theoretical_loss": 3.4718005705197164, + "tokens_seen": 1735393280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822828282828283, + "loss": 1.6192, + "theoretical_loss": 3.4717893316446933, + "tokens_seen": 1735458816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822626262626263, + "loss": 1.277, + "theoretical_loss": 3.471778093312905, + "tokens_seen": 1735524352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048224242424242425, + "loss": 1.3975, + "theoretical_loss": 3.4717668555243053, + "tokens_seen": 1735589888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822222222222222, + "loss": 1.5485, + "theoretical_loss": 3.4717556182788476, + "tokens_seen": 1735655424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822020202020202, + "loss": 1.1766, + "theoretical_loss": 3.471744381576485, + "tokens_seen": 1735720960 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821818181818182, + "loss": 1.3168, + "theoretical_loss": 3.4717331454171703, + "tokens_seen": 1735786496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821616161616162, + "loss": 1.4985, + "theoretical_loss": 3.471721909800857, + "tokens_seen": 1735852032 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048214141414141415, + "loss": 1.4077, + "theoretical_loss": 3.4717106747274986, + "tokens_seen": 1735917568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821212121212121, + "loss": 1.4942, + "theoretical_loss": 3.471699440197048, + "tokens_seen": 1735983104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821010101010101, + "loss": 1.5322, + "theoretical_loss": 3.4716882062094587, + "tokens_seen": 1736048640 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048208080808080807, + "loss": 1.4188, + "theoretical_loss": 3.4716769727646835, + "tokens_seen": 1736114176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820606060606061, + "loss": 1.5154, + "theoretical_loss": 3.4716657398626767, + "tokens_seen": 1736179712 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048204040404040404, + "loss": 1.2208, + "theoretical_loss": 3.471654507503391, + "tokens_seen": 1736245248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048202020202020205, + "loss": 1.524, + "theoretical_loss": 3.4716432756867794, + "tokens_seen": 1736310784 + }, + { + "epoch": 0.05, + "learning_rate": 0.000482, + "loss": 1.5759, + "theoretical_loss": 3.4716320444127957, + "tokens_seen": 1736376320 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048197979797979796, + "loss": 1.3482, + "theoretical_loss": 3.471620813681393, + "tokens_seen": 1736441856 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481959595959596, + "loss": 1.4318, + "theoretical_loss": 3.4716095834925245, + "tokens_seen": 1736507392 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048193939393939393, + "loss": 1.423, + "theoretical_loss": 3.4715983538461432, + "tokens_seen": 1736572928 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048191919191919194, + "loss": 1.4259, + "theoretical_loss": 3.4715871247422037, + "tokens_seen": 1736638464 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818989898989899, + "loss": 1.6113, + "theoretical_loss": 3.4715758961806573, + "tokens_seen": 1736704000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818787878787879, + "loss": 1.4458, + "theoretical_loss": 3.4715646681614594, + "tokens_seen": 1736769536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048185858585858587, + "loss": 1.1604, + "theoretical_loss": 3.4715534406845627, + "tokens_seen": 1736835072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818383838383839, + "loss": 1.3382, + "theoretical_loss": 3.4715422137499194, + "tokens_seen": 1736900608 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.43413978815078735, + "objective/train/avg_token_score": 0.012802019715309143, + "objective/train/docs_used": 983872, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4975614547729492, + "objective/train/original_loss": 4.492546081542969, + "objective/train/theoretical_loss": 3.4715309873574847, + "objective/train/tokens_used": 95825376, + "objective/train/unlikelihood_loss": 0.04530109092593193, + "theoretical_loss": 3.4715309873574847, + "tokens_seen": 1736966144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048181818181818184, + "loss": 1.3615, + "theoretical_loss": 3.4715309873574847, + "tokens_seen": 1736966144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817979797979798, + "loss": 1.5586, + "theoretical_loss": 3.4715197615072104, + "tokens_seen": 1737031680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817777777777778, + "loss": 1.5499, + "theoretical_loss": 3.4715085361990505, + "tokens_seen": 1737097216 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048175757575757576, + "loss": 1.3085, + "theoretical_loss": 3.471497311432959, + "tokens_seen": 1737162752 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048173737373737377, + "loss": 1.4672, + "theoretical_loss": 3.4714860872088877, + "tokens_seen": 1737228288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048171717171717173, + "loss": 1.3109, + "theoretical_loss": 3.471474863526791, + "tokens_seen": 1737293824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816969696969697, + "loss": 1.5911, + "theoretical_loss": 3.471463640386623, + "tokens_seen": 1737359360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816767676767677, + "loss": 1.4829, + "theoretical_loss": 3.4714524177883352, + "tokens_seen": 1737424896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816565656565657, + "loss": 1.3043, + "theoretical_loss": 3.4714411957318827, + "tokens_seen": 1737490432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048163636363636366, + "loss": 1.5117, + "theoretical_loss": 3.4714299742172186, + "tokens_seen": 1737555968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816161616161616, + "loss": 1.5361, + "theoretical_loss": 3.4714187532442953, + "tokens_seen": 1737621504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815959595959596, + "loss": 1.4369, + "theoretical_loss": 3.471407532813067, + "tokens_seen": 1737687040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815757575757576, + "loss": 1.3458, + "theoretical_loss": 3.4713963129234875, + "tokens_seen": 1737752576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815555555555556, + "loss": 1.4904, + "theoretical_loss": 3.4713850935755093, + "tokens_seen": 1737818112 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048153535353535356, + "loss": 1.3929, + "theoretical_loss": 3.4713738747690863, + "tokens_seen": 1737883648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815151515151515, + "loss": 1.5227, + "theoretical_loss": 3.471362656504172, + "tokens_seen": 1737949184 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048149494949494947, + "loss": 1.4509, + "theoretical_loss": 3.4713514387807196, + "tokens_seen": 1738014720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048147474747474754, + "loss": 1.3549, + "theoretical_loss": 3.471340221598683, + "tokens_seen": 1738080256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004814545454545455, + "loss": 1.4395, + "theoretical_loss": 3.4713290049580157, + "tokens_seen": 1738145792 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048143434343434345, + "loss": 1.484, + "theoretical_loss": 3.4713177888586704, + "tokens_seen": 1738211328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004814141414141414, + "loss": 1.3216, + "theoretical_loss": 3.471306573300601, + "tokens_seen": 1738276864 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048139393939393936, + "loss": 1.6081, + "theoretical_loss": 3.4712953582837613, + "tokens_seen": 1738342400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048137373737373743, + "loss": 1.5769, + "theoretical_loss": 3.471284143808104, + "tokens_seen": 1738407936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813535353535354, + "loss": 1.4718, + "theoretical_loss": 3.4712729298735834, + "tokens_seen": 1738473472 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048133333333333334, + "loss": 1.3028, + "theoretical_loss": 3.471261716480152, + "tokens_seen": 1738539008 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.48277124762535095, + "objective/train/avg_token_score": 0.009705035015940666, + "objective/train/docs_used": 984348, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.655248999595642, + "objective/train/original_loss": 4.150419235229492, + "objective/train/theoretical_loss": 3.4712505036277648, + "objective/train/tokens_used": 97463776, + "objective/train/unlikelihood_loss": 0.052979037165641785, + "theoretical_loss": 3.4712505036277648, + "tokens_seen": 1738604544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813131313131313, + "loss": 1.4086, + "theoretical_loss": 3.4712505036277648, + "tokens_seen": 1738604544 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048129292929292926, + "loss": 1.3672, + "theoretical_loss": 3.4712392913163734, + "tokens_seen": 1738670080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812727272727273, + "loss": 1.4242, + "theoretical_loss": 3.4712280795459334, + "tokens_seen": 1738735616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812525252525253, + "loss": 1.5003, + "theoretical_loss": 3.4712168683163966, + "tokens_seen": 1738801152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048123232323232323, + "loss": 1.3127, + "theoretical_loss": 3.4712056576277175, + "tokens_seen": 1738866688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812121212121212, + "loss": 1.3313, + "theoretical_loss": 3.4711944474798493, + "tokens_seen": 1738932224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811919191919192, + "loss": 1.3412, + "theoretical_loss": 3.4711832378727454, + "tokens_seen": 1738997760 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811717171717172, + "loss": 1.3733, + "theoretical_loss": 3.4711720288063592, + "tokens_seen": 1739063296 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048115151515151517, + "loss": 1.4347, + "theoretical_loss": 3.4711608202806445, + "tokens_seen": 1739128832 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048113131313131313, + "loss": 1.1575, + "theoretical_loss": 3.471149612295555, + "tokens_seen": 1739194368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811111111111111, + "loss": 1.4967, + "theoretical_loss": 3.4711384048510445, + "tokens_seen": 1739259904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810909090909091, + "loss": 1.5965, + "theoretical_loss": 3.471127197947066, + "tokens_seen": 1739325440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810707070707071, + "loss": 1.6453, + "theoretical_loss": 3.4711159915835728, + "tokens_seen": 1739390976 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048105050505050506, + "loss": 1.4245, + "theoretical_loss": 3.4711047857605193, + "tokens_seen": 1739456512 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481030303030303, + "loss": 1.5783, + "theoretical_loss": 3.4710935804778584, + "tokens_seen": 1739522048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048101010101010103, + "loss": 1.2761, + "theoretical_loss": 3.471082375735544, + "tokens_seen": 1739587584 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480989898989899, + "loss": 1.4248, + "theoretical_loss": 3.47107117153353, + "tokens_seen": 1739653120 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480969696969697, + "loss": 1.6432, + "theoretical_loss": 3.471059967871769, + "tokens_seen": 1739718656 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048094949494949496, + "loss": 1.394, + "theoretical_loss": 3.4710487647502157, + "tokens_seen": 1739784192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004809292929292929, + "loss": 1.3901, + "theoretical_loss": 3.4710375621688234, + "tokens_seen": 1739849728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004809090909090909, + "loss": 1.4413, + "theoretical_loss": 3.471026360127545, + "tokens_seen": 1739915264 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808888888888889, + "loss": 1.4342, + "theoretical_loss": 3.471015158626335, + "tokens_seen": 1739980800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808686868686869, + "loss": 1.2352, + "theoretical_loss": 3.471003957665147, + "tokens_seen": 1740046336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048084848484848485, + "loss": 1.2348, + "theoretical_loss": 3.470992757243934, + "tokens_seen": 1740111872 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048082828282828286, + "loss": 1.3523, + "theoretical_loss": 3.4709815573626495, + "tokens_seen": 1740177408 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.33125612139701843, + "objective/train/avg_token_score": 0.017105352133512497, + "objective/train/docs_used": 985100, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2295209169387817, + "objective/train/original_loss": 4.453027248382568, + "objective/train/theoretical_loss": 3.4709703580212485, + "objective/train/tokens_used": 99102176, + "objective/train/unlikelihood_loss": 0.0723608136177063, + "theoretical_loss": 3.4709703580212485, + "tokens_seen": 1740242944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808080808080808, + "loss": 1.4608, + "theoretical_loss": 3.4709703580212485, + "tokens_seen": 1740242944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807878787878788, + "loss": 1.4108, + "theoretical_loss": 3.470959159219683, + "tokens_seen": 1740308480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807676767676768, + "loss": 1.5332, + "theoretical_loss": 3.4709479609579077, + "tokens_seen": 1740374016 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048074747474747474, + "loss": 1.7023, + "theoretical_loss": 3.470936763235876, + "tokens_seen": 1740439552 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048072727272727275, + "loss": 1.24, + "theoretical_loss": 3.470925566053541, + "tokens_seen": 1740505088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807070707070707, + "loss": 1.4697, + "theoretical_loss": 3.4709143694108575, + "tokens_seen": 1740570624 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048068686868686867, + "loss": 1.4578, + "theoretical_loss": 3.4709031733077786, + "tokens_seen": 1740636160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806666666666667, + "loss": 1.5702, + "theoretical_loss": 3.4708919777442575, + "tokens_seen": 1740701696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806464646464647, + "loss": 1.5591, + "theoretical_loss": 3.470880782720249, + "tokens_seen": 1740767232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048062626262626264, + "loss": 1.2108, + "theoretical_loss": 3.470869588235705, + "tokens_seen": 1740832768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806060606060606, + "loss": 1.3671, + "theoretical_loss": 3.4708583942905813, + "tokens_seen": 1740898304 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048058585858585856, + "loss": 1.5984, + "theoretical_loss": 3.4708472008848297, + "tokens_seen": 1740963840 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048056565656565657, + "loss": 1.5255, + "theoretical_loss": 3.4708360080184057, + "tokens_seen": 1741029376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805454545454546, + "loss": 1.5005, + "theoretical_loss": 3.4708248156912616, + "tokens_seen": 1741094912 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048052525252525254, + "loss": 1.5318, + "theoretical_loss": 3.470813623903352, + "tokens_seen": 1741160448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805050505050505, + "loss": 1.3555, + "theoretical_loss": 3.47080243265463, + "tokens_seen": 1741225984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048048484848484845, + "loss": 1.324, + "theoretical_loss": 3.47079124194505, + "tokens_seen": 1741291520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804646464646465, + "loss": 1.4643, + "theoretical_loss": 3.4707800517745646, + "tokens_seen": 1741357056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804444444444445, + "loss": 1.2959, + "theoretical_loss": 3.470768862143129, + "tokens_seen": 1741422592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048042424242424243, + "loss": 1.3676, + "theoretical_loss": 3.470757673050696, + "tokens_seen": 1741488128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804040404040404, + "loss": 1.4532, + "theoretical_loss": 3.4707464844972193, + "tokens_seen": 1741553664 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048038383838383834, + "loss": 1.3073, + "theoretical_loss": 3.470735296482653, + "tokens_seen": 1741619200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803636363636364, + "loss": 1.3783, + "theoretical_loss": 3.470724109006951, + "tokens_seen": 1741684736 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048034343434343437, + "loss": 1.6342, + "theoretical_loss": 3.470712922070067, + "tokens_seen": 1741750272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803232323232323, + "loss": 1.5602, + "theoretical_loss": 3.4707017356719545, + "tokens_seen": 1741815808 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.4005987346172333, + "objective/train/avg_token_score": 0.019556434825062752, + "objective/train/docs_used": 986266, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.536611795425415, + "objective/train/original_loss": 4.283116817474365, + "objective/train/theoretical_loss": 3.4706905498125673, + "objective/train/tokens_used": 100740576, + "objective/train/unlikelihood_loss": 0.08906306326389313, + "theoretical_loss": 3.4706905498125673, + "tokens_seen": 1741881344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803030303030303, + "loss": 1.4406, + "theoretical_loss": 3.4706905498125673, + "tokens_seen": 1741881344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802828282828283, + "loss": 1.4997, + "theoretical_loss": 3.47067936449186, + "tokens_seen": 1741946880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802626262626263, + "loss": 1.6152, + "theoretical_loss": 3.4706681797097847, + "tokens_seen": 1742012416 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048024242424242426, + "loss": 1.5295, + "theoretical_loss": 3.470656995466297, + "tokens_seen": 1742077952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802222222222222, + "loss": 1.3141, + "theoretical_loss": 3.47064581176135, + "tokens_seen": 1742143488 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048020202020202017, + "loss": 1.3606, + "theoretical_loss": 3.470634628594897, + "tokens_seen": 1742209024 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801818181818182, + "loss": 1.4034, + "theoretical_loss": 3.4706234459668925, + "tokens_seen": 1742274560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801616161616162, + "loss": 1.4074, + "theoretical_loss": 3.47061226387729, + "tokens_seen": 1742340096 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048014141414141415, + "loss": 1.2829, + "theoretical_loss": 3.470601082326043, + "tokens_seen": 1742405632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801212121212121, + "loss": 1.4758, + "theoretical_loss": 3.4705899013131063, + "tokens_seen": 1742471168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801010101010101, + "loss": 1.3614, + "theoretical_loss": 3.470578720838433, + "tokens_seen": 1742536704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048008080808080813, + "loss": 1.3864, + "theoretical_loss": 3.4705675409019774, + "tokens_seen": 1742602240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800606060606061, + "loss": 1.4217, + "theoretical_loss": 3.4705563615036925, + "tokens_seen": 1742667776 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048004040404040404, + "loss": 1.1732, + "theoretical_loss": 3.4705451826435327, + "tokens_seen": 1742733312 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480020202020202, + "loss": 1.4523, + "theoretical_loss": 3.4705340043214523, + "tokens_seen": 1742798848 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048, + "loss": 1.4718, + "theoretical_loss": 3.4705228265374046, + "tokens_seen": 1742864384 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479979797979798, + "loss": 1.4891, + "theoretical_loss": 3.470511649291344, + "tokens_seen": 1742929920 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479959595959596, + "loss": 1.2984, + "theoretical_loss": 3.4705004725832236, + "tokens_seen": 1742995456 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047993939393939394, + "loss": 1.5396, + "theoretical_loss": 3.470489296412998, + "tokens_seen": 1743060992 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047991919191919195, + "loss": 1.6143, + "theoretical_loss": 3.470478120780621, + "tokens_seen": 1743126528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798989898989899, + "loss": 1.3893, + "theoretical_loss": 3.4704669456860455, + "tokens_seen": 1743192064 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798787878787879, + "loss": 1.3921, + "theoretical_loss": 3.4704557711292265, + "tokens_seen": 1743257600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047985858585858587, + "loss": 1.326, + "theoretical_loss": 3.470444597110118, + "tokens_seen": 1743323136 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047983838383838383, + "loss": 1.3948, + "theoretical_loss": 3.4704334236286734, + "tokens_seen": 1743388672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047981818181818184, + "loss": 1.3807, + "theoretical_loss": 3.4704222506848463, + "tokens_seen": 1743454208 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.3908846378326416, + "objective/train/avg_token_score": 0.00899689830839634, + "objective/train/docs_used": 986906, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2534172534942627, + "objective/train/original_loss": 4.556232929229736, + "objective/train/theoretical_loss": 3.470411078278591, + "objective/train/tokens_used": 102378976, + "objective/train/unlikelihood_loss": 0.03879563882946968, + "theoretical_loss": 3.470411078278591, + "tokens_seen": 1743519744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797979797979798, + "loss": 1.5736, + "theoretical_loss": 3.470411078278591, + "tokens_seen": 1743519744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797777777777778, + "loss": 1.304, + "theoretical_loss": 3.4703999064098623, + "tokens_seen": 1743585280 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047975757575757577, + "loss": 1.4849, + "theoretical_loss": 3.470388735078613, + "tokens_seen": 1743650816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797373737373738, + "loss": 1.3432, + "theoretical_loss": 3.4703775642847967, + "tokens_seen": 1743716352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047971717171717173, + "loss": 1.1625, + "theoretical_loss": 3.4703663940283684, + "tokens_seen": 1743781888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796969696969697, + "loss": 1.6368, + "theoretical_loss": 3.470355224309282, + "tokens_seen": 1743847424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796767676767677, + "loss": 1.4579, + "theoretical_loss": 3.470344055127491, + "tokens_seen": 1743912960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047965656565656566, + "loss": 1.4415, + "theoretical_loss": 3.4703328864829492, + "tokens_seen": 1743978496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047963636363636367, + "loss": 1.5911, + "theoretical_loss": 3.4703217183756108, + "tokens_seen": 1744044032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796161616161616, + "loss": 1.2517, + "theoretical_loss": 3.47031055080543, + "tokens_seen": 1744109568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795959595959596, + "loss": 1.2024, + "theoretical_loss": 3.470299383772361, + "tokens_seen": 1744175104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795757575757576, + "loss": 1.3713, + "theoretical_loss": 3.470288217276357, + "tokens_seen": 1744240640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795555555555556, + "loss": 1.4253, + "theoretical_loss": 3.4702770513173724, + "tokens_seen": 1744306176 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047953535353535356, + "loss": 1.406, + "theoretical_loss": 3.470265885895361, + "tokens_seen": 1744371712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795151515151515, + "loss": 1.4899, + "theoretical_loss": 3.4702547210102774, + "tokens_seen": 1744437248 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794949494949495, + "loss": 1.5386, + "theoretical_loss": 3.470243556662075, + "tokens_seen": 1744502784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794747474747475, + "loss": 1.2509, + "theoretical_loss": 3.470232392850708, + "tokens_seen": 1744568320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794545454545455, + "loss": 1.7622, + "theoretical_loss": 3.470221229576131, + "tokens_seen": 1744633856 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047943434343434345, + "loss": 1.3678, + "theoretical_loss": 3.4702100668382965, + "tokens_seen": 1744699392 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794141414141414, + "loss": 1.4218, + "theoretical_loss": 3.47019890463716, + "tokens_seen": 1744764928 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047939393939393937, + "loss": 1.4213, + "theoretical_loss": 3.470187742972675, + "tokens_seen": 1744830464 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793737373737374, + "loss": 1.4785, + "theoretical_loss": 3.470176581844796, + "tokens_seen": 1744896000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793535353535354, + "loss": 1.5449, + "theoretical_loss": 3.4701654212534763, + "tokens_seen": 1744961536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047933333333333335, + "loss": 1.287, + "theoretical_loss": 3.4701542611986698, + "tokens_seen": 1745027072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793131313131313, + "loss": 1.567, + "theoretical_loss": 3.470143101680332, + "tokens_seen": 1745092608 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.2597140669822693, + "objective/train/avg_token_score": 0.04489205777645111, + "objective/train/docs_used": 988319, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.912868082523346, + "objective/train/original_loss": 4.443542003631592, + "objective/train/theoretical_loss": 3.4701319426984156, + "objective/train/tokens_used": 104017376, + "objective/train/unlikelihood_loss": 0.06317847967147827, + "theoretical_loss": 3.4701319426984156, + "tokens_seen": 1745158144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047929292929292926, + "loss": 1.2568, + "theoretical_loss": 3.4701319426984156, + "tokens_seen": 1745158144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792727272727273, + "loss": 1.3641, + "theoretical_loss": 3.4701207842528747, + "tokens_seen": 1745223680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792525252525253, + "loss": 1.5416, + "theoretical_loss": 3.470109626343664, + "tokens_seen": 1745289216 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047923232323232324, + "loss": 1.4213, + "theoretical_loss": 3.470098468970738, + "tokens_seen": 1745354752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792121212121212, + "loss": 1.0977, + "theoretical_loss": 3.470087312134049, + "tokens_seen": 1745420288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047919191919191915, + "loss": 1.5069, + "theoretical_loss": 3.470076155833553, + "tokens_seen": 1745485824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791717171717172, + "loss": 1.569, + "theoretical_loss": 3.4700650000692033, + "tokens_seen": 1745551360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791515151515152, + "loss": 1.5586, + "theoretical_loss": 3.4700538448409537, + "tokens_seen": 1745616896 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047913131313131313, + "loss": 1.5639, + "theoretical_loss": 3.470042690148759, + "tokens_seen": 1745682432 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791111111111111, + "loss": 1.4329, + "theoretical_loss": 3.4700315359925726, + "tokens_seen": 1745747968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790909090909091, + "loss": 1.4756, + "theoretical_loss": 3.470020382372349, + "tokens_seen": 1745813504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790707070707071, + "loss": 1.4192, + "theoretical_loss": 3.470009229288043, + "tokens_seen": 1745879040 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047905050505050507, + "loss": 1.2349, + "theoretical_loss": 3.469998076739607, + "tokens_seen": 1745944576 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479030303030303, + "loss": 1.4639, + "theoretical_loss": 3.469986924726997, + "tokens_seen": 1746010112 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479010101010101, + "loss": 1.5232, + "theoretical_loss": 3.469975773250166, + "tokens_seen": 1746075648 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478989898989899, + "loss": 1.1636, + "theoretical_loss": 3.4699646223090688, + "tokens_seen": 1746141184 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478969696969697, + "loss": 1.4086, + "theoretical_loss": 3.469953471903659, + "tokens_seen": 1746206720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047894949494949496, + "loss": 1.3567, + "theoretical_loss": 3.469942322033891, + "tokens_seen": 1746272256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789292929292929, + "loss": 1.6627, + "theoretical_loss": 3.4699311726997193, + "tokens_seen": 1746337792 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047890909090909093, + "loss": 1.4643, + "theoretical_loss": 3.469920023901097, + "tokens_seen": 1746403328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788888888888889, + "loss": 1.393, + "theoretical_loss": 3.46990887563798, + "tokens_seen": 1746468864 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788686868686869, + "loss": 1.6576, + "theoretical_loss": 3.469897727910321, + "tokens_seen": 1746534400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047884848484848485, + "loss": 1.3613, + "theoretical_loss": 3.4698865807180748, + "tokens_seen": 1746599936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788282828282828, + "loss": 1.5132, + "theoretical_loss": 3.4698754340611955, + "tokens_seen": 1746665472 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788080808080808, + "loss": 1.4484, + "theoretical_loss": 3.4698642879396373, + "tokens_seen": 1746731008 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.29673752188682556, + "objective/train/avg_token_score": 0.007496143225580454, + "objective/train/docs_used": 989102, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9727373123168945, + "objective/train/original_loss": 4.123487949371338, + "objective/train/theoretical_loss": 3.4698531423533545, + "objective/train/tokens_used": 105655776, + "objective/train/unlikelihood_loss": 0.07145555317401886, + "theoretical_loss": 3.4698531423533545, + "tokens_seen": 1746796544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787878787878788, + "loss": 1.2235, + "theoretical_loss": 3.4698531423533545, + "tokens_seen": 1746796544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787676767676768, + "loss": 1.4326, + "theoretical_loss": 3.469841997302301, + "tokens_seen": 1746862080 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047874747474747475, + "loss": 1.4788, + "theoretical_loss": 3.4698308527864317, + "tokens_seen": 1746927616 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047872727272727276, + "loss": 1.4569, + "theoretical_loss": 3.4698197088056997, + "tokens_seen": 1746993152 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787070707070707, + "loss": 1.3571, + "theoretical_loss": 3.4698085653600605, + "tokens_seen": 1747058688 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047868686868686867, + "loss": 1.3078, + "theoretical_loss": 3.469797422449467, + "tokens_seen": 1747124224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786666666666667, + "loss": 1.3261, + "theoretical_loss": 3.4697862800738752, + "tokens_seen": 1747189760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047864646464646464, + "loss": 1.5378, + "theoretical_loss": 3.4697751382332376, + "tokens_seen": 1747255296 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047862626262626265, + "loss": 1.3303, + "theoretical_loss": 3.46976399692751, + "tokens_seen": 1747320832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786060606060606, + "loss": 1.513, + "theoretical_loss": 3.469752856156645, + "tokens_seen": 1747386368 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047858585858585856, + "loss": 1.5127, + "theoretical_loss": 3.469741715920598, + "tokens_seen": 1747451904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785656565656566, + "loss": 1.511, + "theoretical_loss": 3.4697305762193227, + "tokens_seen": 1747517440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785454545454546, + "loss": 1.3687, + "theoretical_loss": 3.469719437052774, + "tokens_seen": 1747582976 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047852525252525254, + "loss": 1.2167, + "theoretical_loss": 3.4697082984209056, + "tokens_seen": 1747648512 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785050505050505, + "loss": 1.3243, + "theoretical_loss": 3.469697160323672, + "tokens_seen": 1747714048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047848484848484846, + "loss": 1.7173, + "theoretical_loss": 3.469686022761028, + "tokens_seen": 1747779584 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047846464646464647, + "loss": 1.2967, + "theoretical_loss": 3.469674885732927, + "tokens_seen": 1747845120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784444444444445, + "loss": 1.3806, + "theoretical_loss": 3.4696637492393236, + "tokens_seen": 1747910656 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047842424242424244, + "loss": 1.433, + "theoretical_loss": 3.4696526132801724, + "tokens_seen": 1747976192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784040404040404, + "loss": 1.2802, + "theoretical_loss": 3.4696414778554274, + "tokens_seen": 1748041728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783838383838384, + "loss": 1.3713, + "theoretical_loss": 3.4696303429650435, + "tokens_seen": 1748107264 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783636363636364, + "loss": 1.288, + "theoretical_loss": 3.469619208608974, + "tokens_seen": 1748172800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047834343434343437, + "loss": 1.5354, + "theoretical_loss": 3.469608074787174, + "tokens_seen": 1748238336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047832323232323233, + "loss": 1.3335, + "theoretical_loss": 3.4695969414995975, + "tokens_seen": 1748303872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783030303030303, + "loss": 1.2709, + "theoretical_loss": 3.4695858087461993, + "tokens_seen": 1748369408 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.356671541929245, + "objective/train/avg_token_score": 0.015191304497420788, + "objective/train/docs_used": 990185, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0629782676696777, + "objective/train/original_loss": 4.2517290115356445, + "objective/train/theoretical_loss": 3.469574676526933, + "objective/train/tokens_used": 107294176, + "objective/train/unlikelihood_loss": 0.06087062880396843, + "theoretical_loss": 3.469574676526933, + "tokens_seen": 1748434944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782828282828283, + "loss": 1.27, + "theoretical_loss": 3.469574676526933, + "tokens_seen": 1748434944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782626262626263, + "loss": 1.3251, + "theoretical_loss": 3.469563544841754, + "tokens_seen": 1748500480 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047824242424242426, + "loss": 1.374, + "theoretical_loss": 3.4695524136906153, + "tokens_seen": 1748566016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782222222222222, + "loss": 1.3274, + "theoretical_loss": 3.4695412830734726, + "tokens_seen": 1748631552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782020202020202, + "loss": 1.5862, + "theoretical_loss": 3.46953015299028, + "tokens_seen": 1748697088 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047818181818181824, + "loss": 1.3369, + "theoretical_loss": 3.469519023440991, + "tokens_seen": 1748762624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781616161616162, + "loss": 1.5887, + "theoretical_loss": 3.4695078944255604, + "tokens_seen": 1748828160 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047814141414141416, + "loss": 1.4068, + "theoretical_loss": 3.4694967659439433, + "tokens_seen": 1748893696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781212121212121, + "loss": 1.4508, + "theoretical_loss": 3.469485637996093, + "tokens_seen": 1748959232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047810101010101007, + "loss": 1.3742, + "theoretical_loss": 3.4694745105819647, + "tokens_seen": 1749024768 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047808080808080814, + "loss": 1.255, + "theoretical_loss": 3.4694633837015125, + "tokens_seen": 1749090304 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780606060606061, + "loss": 1.3452, + "theoretical_loss": 3.469452257354691, + "tokens_seen": 1749155840 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047804040404040405, + "loss": 1.4228, + "theoretical_loss": 3.469441131541454, + "tokens_seen": 1749221376 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478020202020202, + "loss": 1.3645, + "theoretical_loss": 3.4694300062617565, + "tokens_seen": 1749286912 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047799999999999996, + "loss": 1.2568, + "theoretical_loss": 3.469418881515553, + "tokens_seen": 1749352448 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047797979797979803, + "loss": 1.2278, + "theoretical_loss": 3.4694077573027977, + "tokens_seen": 1749417984 + }, + { + "epoch": 0.05, + "learning_rate": 0.000477959595959596, + "loss": 1.3498, + "theoretical_loss": 3.469396633623445, + "tokens_seen": 1749483520 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047793939393939394, + "loss": 1.201, + "theoretical_loss": 3.4693855104774496, + "tokens_seen": 1749549056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779191919191919, + "loss": 1.4381, + "theoretical_loss": 3.4693743878647654, + "tokens_seen": 1749614592 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778989898989899, + "loss": 1.3039, + "theoretical_loss": 3.469363265785348, + "tokens_seen": 1749680128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778787878787879, + "loss": 1.6264, + "theoretical_loss": 3.46935214423915, + "tokens_seen": 1749745664 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778585858585859, + "loss": 1.6328, + "theoretical_loss": 3.4693410232261277, + "tokens_seen": 1749811200 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047783838383838384, + "loss": 1.5476, + "theoretical_loss": 3.4693299027462343, + "tokens_seen": 1749876736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778181818181818, + "loss": 1.2902, + "theoretical_loss": 3.469318782799425, + "tokens_seen": 1749942272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777979797979798, + "loss": 1.4278, + "theoretical_loss": 3.4693076633856545, + "tokens_seen": 1750007808 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.4995723366737366, + "objective/train/avg_token_score": 0.01636279560625553, + "objective/train/docs_used": 990903, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7601079940795898, + "objective/train/original_loss": 4.128569602966309, + "objective/train/theoretical_loss": 3.4692965445048767, + "objective/train/tokens_used": 108932576, + "objective/train/unlikelihood_loss": 0.05954797938466072, + "theoretical_loss": 3.4692965445048767, + "tokens_seen": 1750073344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777777777777778, + "loss": 1.532, + "theoretical_loss": 3.4692965445048767, + "tokens_seen": 1750073344 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047775757575757577, + "loss": 1.3282, + "theoretical_loss": 3.469285426157046, + "tokens_seen": 1750138880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047773737373737373, + "loss": 1.3803, + "theoretical_loss": 3.469274308342117, + "tokens_seen": 1750204416 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047771717171717174, + "loss": 1.5482, + "theoretical_loss": 3.469263191060045, + "tokens_seen": 1750269952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776969696969697, + "loss": 1.6532, + "theoretical_loss": 3.4692520743107833, + "tokens_seen": 1750335488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776767676767677, + "loss": 1.4167, + "theoretical_loss": 3.469240958094287, + "tokens_seen": 1750401024 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047765656565656566, + "loss": 1.193, + "theoretical_loss": 3.469229842410511, + "tokens_seen": 1750466560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776363636363636, + "loss": 1.3584, + "theoretical_loss": 3.4692187272594093, + "tokens_seen": 1750532096 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047761616161616163, + "loss": 1.439, + "theoretical_loss": 3.469207612640936, + "tokens_seen": 1750597632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775959595959596, + "loss": 1.2879, + "theoretical_loss": 3.469196498555047, + "tokens_seen": 1750663168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775757575757576, + "loss": 1.3754, + "theoretical_loss": 3.469185385001696, + "tokens_seen": 1750728704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047755555555555556, + "loss": 1.2025, + "theoretical_loss": 3.4691742719808376, + "tokens_seen": 1750794240 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047753535353535357, + "loss": 1.5471, + "theoretical_loss": 3.469163159492426, + "tokens_seen": 1750859776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775151515151515, + "loss": 1.2861, + "theoretical_loss": 3.469152047536416, + "tokens_seen": 1750925312 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774949494949495, + "loss": 1.4033, + "theoretical_loss": 3.4691409361127628, + "tokens_seen": 1750990848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774747474747475, + "loss": 1.3833, + "theoretical_loss": 3.46912982522142, + "tokens_seen": 1751056384 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047745454545454545, + "loss": 1.5292, + "theoretical_loss": 3.469118714862343, + "tokens_seen": 1751121920 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047743434343434346, + "loss": 1.3997, + "theoretical_loss": 3.469107605035486, + "tokens_seen": 1751187456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774141414141414, + "loss": 1.3918, + "theoretical_loss": 3.4690964957408035, + "tokens_seen": 1751252992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773939393939394, + "loss": 1.5265, + "theoretical_loss": 3.46908538697825, + "tokens_seen": 1751318528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773737373737374, + "loss": 1.2903, + "theoretical_loss": 3.4690742787477804, + "tokens_seen": 1751384064 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773535353535354, + "loss": 1.4772, + "theoretical_loss": 3.4690631710493496, + "tokens_seen": 1751449600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047733333333333335, + "loss": 1.4411, + "theoretical_loss": 3.469052063882912, + "tokens_seen": 1751515136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773131313131313, + "loss": 1.4693, + "theoretical_loss": 3.4690409572484215, + "tokens_seen": 1751580672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047729292929292927, + "loss": 1.6307, + "theoretical_loss": 3.469029851145833, + "tokens_seen": 1751646208 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.35160067677497864, + "objective/train/avg_token_score": 0.0432390458881855, + "objective/train/docs_used": 991641, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.203386664390564, + "objective/train/original_loss": 4.336913108825684, + "objective/train/theoretical_loss": 3.469018745575102, + "objective/train/tokens_used": 110570976, + "objective/train/unlikelihood_loss": 0.058832619339227676, + "theoretical_loss": 3.469018745575102, + "tokens_seen": 1751711744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772727272727273, + "loss": 1.3743, + "theoretical_loss": 3.469018745575102, + "tokens_seen": 1751711744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772525252525253, + "loss": 1.4335, + "theoretical_loss": 3.469007640536182, + "tokens_seen": 1751777280 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047723232323232325, + "loss": 1.5358, + "theoretical_loss": 3.4689965360290285, + "tokens_seen": 1751842816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772121212121212, + "loss": 1.2956, + "theoretical_loss": 3.4689854320535956, + "tokens_seen": 1751908352 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047719191919191916, + "loss": 1.5891, + "theoretical_loss": 3.4689743286098382, + "tokens_seen": 1751973888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771717171717172, + "loss": 1.5675, + "theoretical_loss": 3.468963225697711, + "tokens_seen": 1752039424 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771515151515152, + "loss": 1.3356, + "theoretical_loss": 3.4689521233171687, + "tokens_seen": 1752104960 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047713131313131314, + "loss": 1.4772, + "theoretical_loss": 3.468941021468166, + "tokens_seen": 1752170496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771111111111111, + "loss": 1.2844, + "theoretical_loss": 3.468929920150657, + "tokens_seen": 1752236032 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047709090909090905, + "loss": 1.5272, + "theoretical_loss": 3.468918819364597, + "tokens_seen": 1752301568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770707070707071, + "loss": 1.4227, + "theoretical_loss": 3.4689077191099402, + "tokens_seen": 1752367104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770505050505051, + "loss": 1.2629, + "theoretical_loss": 3.4688966193866415, + "tokens_seen": 1752432640 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047703030303030303, + "loss": 1.4331, + "theoretical_loss": 3.468885520194656, + "tokens_seen": 1752498176 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477010101010101, + "loss": 1.4719, + "theoretical_loss": 3.468874421533938, + "tokens_seen": 1752563712 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476989898989899, + "loss": 1.4844, + "theoretical_loss": 3.4688633234044426, + "tokens_seen": 1752629248 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476969696969697, + "loss": 1.6083, + "theoretical_loss": 3.468852225806123, + "tokens_seen": 1752694784 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047694949494949497, + "loss": 1.3576, + "theoretical_loss": 3.4688411287389362, + "tokens_seen": 1752760320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769292929292929, + "loss": 1.5143, + "theoretical_loss": 3.4688300322028356, + "tokens_seen": 1752825856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769090909090909, + "loss": 1.4537, + "theoretical_loss": 3.468818936197776, + "tokens_seen": 1752891392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768888888888889, + "loss": 1.5708, + "theoretical_loss": 3.4688078407237124, + "tokens_seen": 1752956928 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768686868686869, + "loss": 1.4414, + "theoretical_loss": 3.4687967457805993, + "tokens_seen": 1753022464 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047684848484848486, + "loss": 1.5843, + "theoretical_loss": 3.4687856513683917, + "tokens_seen": 1753088000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768282828282828, + "loss": 1.5725, + "theoretical_loss": 3.468774557487044, + "tokens_seen": 1753153536 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047680808080808083, + "loss": 1.2515, + "theoretical_loss": 3.4687634641365115, + "tokens_seen": 1753219072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767878787878788, + "loss": 1.7292, + "theoretical_loss": 3.4687523713167483, + "tokens_seen": 1753284608 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.5292032957077026, + "objective/train/avg_token_score": 0.022525779902935028, + "objective/train/docs_used": 992944, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7439391613006592, + "objective/train/original_loss": 3.8954503536224365, + "objective/train/theoretical_loss": 3.468741279027709, + "objective/train/tokens_used": 112209376, + "objective/train/unlikelihood_loss": 0.06822893768548965, + "theoretical_loss": 3.468741279027709, + "tokens_seen": 1753350144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767676767676768, + "loss": 1.5733, + "theoretical_loss": 3.468741279027709, + "tokens_seen": 1753350144 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047674747474747475, + "loss": 1.3831, + "theoretical_loss": 3.4687301872693497, + "tokens_seen": 1753415680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767272727272727, + "loss": 1.2289, + "theoretical_loss": 3.4687190960416245, + "tokens_seen": 1753481216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767070707070707, + "loss": 1.2174, + "theoretical_loss": 3.468708005344487, + "tokens_seen": 1753546752 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047668686868686873, + "loss": 1.4894, + "theoretical_loss": 3.468696915177894, + "tokens_seen": 1753612288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766666666666667, + "loss": 1.7117, + "theoretical_loss": 3.4686858255417987, + "tokens_seen": 1753677824 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047664646464646465, + "loss": 1.5382, + "theoretical_loss": 3.4686747364361565, + "tokens_seen": 1753743360 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047662626262626266, + "loss": 1.6347, + "theoretical_loss": 3.4686636478609225, + "tokens_seen": 1753808896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766060606060606, + "loss": 1.3016, + "theoretical_loss": 3.4686525598160514, + "tokens_seen": 1753874432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765858585858586, + "loss": 1.2402, + "theoretical_loss": 3.4686414723014973, + "tokens_seen": 1753939968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765656565656566, + "loss": 1.6342, + "theoretical_loss": 3.4686303853172165, + "tokens_seen": 1754005504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047654545454545454, + "loss": 1.4743, + "theoretical_loss": 3.468619298863162, + "tokens_seen": 1754071040 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047652525252525255, + "loss": 1.5816, + "theoretical_loss": 3.46860821293929, + "tokens_seen": 1754136576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765050505050505, + "loss": 1.3497, + "theoretical_loss": 3.4685971275455545, + "tokens_seen": 1754202112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764848484848485, + "loss": 1.4686, + "theoretical_loss": 3.4685860426819106, + "tokens_seen": 1754267648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764646464646465, + "loss": 1.5528, + "theoretical_loss": 3.468574958348314, + "tokens_seen": 1754333184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764444444444445, + "loss": 1.4893, + "theoretical_loss": 3.468563874544718, + "tokens_seen": 1754398720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047642424242424244, + "loss": 1.5118, + "theoretical_loss": 3.4685527912710787, + "tokens_seen": 1754464256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764040404040404, + "loss": 1.5134, + "theoretical_loss": 3.4685417085273507, + "tokens_seen": 1754529792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763838383838384, + "loss": 1.6654, + "theoretical_loss": 3.468530626313488, + "tokens_seen": 1754595328 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047636363636363637, + "loss": 1.5252, + "theoretical_loss": 3.468519544629447, + "tokens_seen": 1754660864 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763434343434344, + "loss": 1.2085, + "theoretical_loss": 3.468508463475181, + "tokens_seen": 1754726400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047632323232323233, + "loss": 1.4196, + "theoretical_loss": 3.4684973828506465, + "tokens_seen": 1754791936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763030303030303, + "loss": 1.6456, + "theoretical_loss": 3.468486302755797, + "tokens_seen": 1754857472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762828282828283, + "loss": 1.5107, + "theoretical_loss": 3.4684752231905875, + "tokens_seen": 1754923008 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.3735337257385254, + "objective/train/avg_token_score": 0.013170573860406876, + "objective/train/docs_used": 993452, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3061128854751587, + "objective/train/original_loss": 4.716046333312988, + "objective/train/theoretical_loss": 3.468464144154974, + "objective/train/tokens_used": 113847776, + "objective/train/unlikelihood_loss": 0.05805331468582153, + "theoretical_loss": 3.468464144154974, + "tokens_seen": 1754988544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762626262626263, + "loss": 1.1821, + "theoretical_loss": 3.468464144154974, + "tokens_seen": 1754988544 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047624242424242427, + "loss": 1.3812, + "theoretical_loss": 3.468453065648911, + "tokens_seen": 1755054080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762222222222222, + "loss": 1.5309, + "theoretical_loss": 3.468441987672353, + "tokens_seen": 1755119616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762020202020202, + "loss": 1.5255, + "theoretical_loss": 3.468430910225255, + "tokens_seen": 1755185152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761818181818182, + "loss": 1.4808, + "theoretical_loss": 3.468419833307572, + "tokens_seen": 1755250688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761616161616162, + "loss": 1.5992, + "theoretical_loss": 3.468408756919259, + "tokens_seen": 1755316224 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047614141414141416, + "loss": 1.4779, + "theoretical_loss": 3.468397681060271, + "tokens_seen": 1755381760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761212121212121, + "loss": 1.4515, + "theoretical_loss": 3.4683866057305623, + "tokens_seen": 1755447296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761010101010101, + "loss": 1.3284, + "theoretical_loss": 3.468375530930089, + "tokens_seen": 1755512832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760808080808081, + "loss": 1.4936, + "theoretical_loss": 3.468364456658805, + "tokens_seen": 1755578368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760606060606061, + "loss": 1.4289, + "theoretical_loss": 3.4683533829166664, + "tokens_seen": 1755643904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047604040404040406, + "loss": 1.5279, + "theoretical_loss": 3.468342309703627, + "tokens_seen": 1755709440 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476020202020202, + "loss": 1.1404, + "theoretical_loss": 3.4683312370196426, + "tokens_seen": 1755774976 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047599999999999997, + "loss": 1.6458, + "theoretical_loss": 3.4683201648646675, + "tokens_seen": 1755840512 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047597979797979803, + "loss": 1.3894, + "theoretical_loss": 3.4683090932386573, + "tokens_seen": 1755906048 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475959595959596, + "loss": 1.5657, + "theoretical_loss": 3.4682980221415662, + "tokens_seen": 1755971584 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047593939393939395, + "loss": 1.2969, + "theoretical_loss": 3.46828695157335, + "tokens_seen": 1756037120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759191919191919, + "loss": 1.4629, + "theoretical_loss": 3.4682758815339634, + "tokens_seen": 1756102656 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047589898989898986, + "loss": 1.4144, + "theoretical_loss": 3.4682648120233615, + "tokens_seen": 1756168192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047587878787878793, + "loss": 1.2853, + "theoretical_loss": 3.4682537430414992, + "tokens_seen": 1756233728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758585858585859, + "loss": 1.1362, + "theoretical_loss": 3.468242674588331, + "tokens_seen": 1756299264 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047583838383838384, + "loss": 1.4003, + "theoretical_loss": 3.468231606663813, + "tokens_seen": 1756364800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758181818181818, + "loss": 1.3655, + "theoretical_loss": 3.4682205392678993, + "tokens_seen": 1756430336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757979797979798, + "loss": 1.2837, + "theoretical_loss": 3.4682094724005457, + "tokens_seen": 1756495872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757777777777778, + "loss": 1.2286, + "theoretical_loss": 3.4681984060617066, + "tokens_seen": 1756561408 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.4603494703769684, + "objective/train/avg_token_score": 0.007801194675266743, + "objective/train/docs_used": 994709, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4813201427459717, + "objective/train/original_loss": 3.937505006790161, + "objective/train/theoretical_loss": 3.4681873402513372, + "objective/train/tokens_used": 115486176, + "objective/train/unlikelihood_loss": 0.07865725457668304, + "theoretical_loss": 3.4681873402513372, + "tokens_seen": 1756626944 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757575757575758, + "loss": 1.5432, + "theoretical_loss": 3.4681873402513372, + "tokens_seen": 1756626944 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047573737373737373, + "loss": 1.328, + "theoretical_loss": 3.4681762749693927, + "tokens_seen": 1756692480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757171717171717, + "loss": 1.3619, + "theoretical_loss": 3.4681652102158282, + "tokens_seen": 1756758016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756969696969697, + "loss": 1.2638, + "theoretical_loss": 3.4681541459905985, + "tokens_seen": 1756823552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756767676767677, + "loss": 1.5705, + "theoretical_loss": 3.468143082293659, + "tokens_seen": 1756889088 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047565656565656567, + "loss": 1.3297, + "theoretical_loss": 3.4681320191249645, + "tokens_seen": 1756954624 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756363636363636, + "loss": 1.2614, + "theoretical_loss": 3.46812095648447, + "tokens_seen": 1757020160 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047561616161616164, + "loss": 1.3043, + "theoretical_loss": 3.468109894372131, + "tokens_seen": 1757085696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755959595959596, + "loss": 1.4471, + "theoretical_loss": 3.468098832787902, + "tokens_seen": 1757151232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755757575757576, + "loss": 1.4198, + "theoretical_loss": 3.4680877717317387, + "tokens_seen": 1757216768 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047555555555555556, + "loss": 1.4285, + "theoretical_loss": 3.4680767112035955, + "tokens_seen": 1757282304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755353535353535, + "loss": 1.3098, + "theoretical_loss": 3.4680656512034282, + "tokens_seen": 1757347840 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047551515151515153, + "loss": 1.4135, + "theoretical_loss": 3.4680545917311916, + "tokens_seen": 1757413376 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754949494949495, + "loss": 1.2951, + "theoretical_loss": 3.468043532786841, + "tokens_seen": 1757478912 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754747474747475, + "loss": 1.4952, + "theoretical_loss": 3.4680324743703315, + "tokens_seen": 1757544448 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047545454545454545, + "loss": 1.3478, + "theoretical_loss": 3.4680214164816174, + "tokens_seen": 1757609984 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047543434343434347, + "loss": 1.4044, + "theoretical_loss": 3.468010359120655, + "tokens_seen": 1757675520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754141414141414, + "loss": 1.3726, + "theoretical_loss": 3.467999302287399, + "tokens_seen": 1757741056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753939393939394, + "loss": 1.5165, + "theoretical_loss": 3.4679882459818043, + "tokens_seen": 1757806592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753737373737374, + "loss": 1.6779, + "theoretical_loss": 3.4679771902038263, + "tokens_seen": 1757872128 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047535353535353535, + "loss": 1.3127, + "theoretical_loss": 3.4679661349534197, + "tokens_seen": 1757937664 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047533333333333336, + "loss": 1.4477, + "theoretical_loss": 3.4679550802305408, + "tokens_seen": 1758003200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753131313131313, + "loss": 1.3832, + "theoretical_loss": 3.467944026035144, + "tokens_seen": 1758068736 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047529292929292927, + "loss": 1.3038, + "theoretical_loss": 3.4679329723671835, + "tokens_seen": 1758134272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752727272727273, + "loss": 1.3008, + "theoretical_loss": 3.467921919226616, + "tokens_seen": 1758199808 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.37341153621673584, + "objective/train/avg_token_score": 0.01459649857133627, + "objective/train/docs_used": 995465, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.182531476020813, + "objective/train/original_loss": 4.433279991149902, + "objective/train/theoretical_loss": 3.4679108666133964, + "objective/train/tokens_used": 117124576, + "objective/train/unlikelihood_loss": 0.05860118195414543, + "theoretical_loss": 3.4679108666133964, + "tokens_seen": 1758265344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752525252525253, + "loss": 1.3201, + "theoretical_loss": 3.4679108666133964, + "tokens_seen": 1758265344 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047523232323232325, + "loss": 1.4253, + "theoretical_loss": 3.4678998145274793, + "tokens_seen": 1758330880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752121212121212, + "loss": 1.2983, + "theoretical_loss": 3.467888762968821, + "tokens_seen": 1758396416 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047519191919191916, + "loss": 1.4404, + "theoretical_loss": 3.467877711937375, + "tokens_seen": 1758461952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751717171717172, + "loss": 1.4583, + "theoretical_loss": 3.4678666614330975, + "tokens_seen": 1758527488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751515151515152, + "loss": 1.4141, + "theoretical_loss": 3.467855611455944, + "tokens_seen": 1758593024 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047513131313131314, + "loss": 1.527, + "theoretical_loss": 3.467844562005869, + "tokens_seen": 1758658560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751111111111111, + "loss": 1.3646, + "theoretical_loss": 3.4678335130828284, + "tokens_seen": 1758724096 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047509090909090906, + "loss": 1.2391, + "theoretical_loss": 3.4678224646867766, + "tokens_seen": 1758789632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750707070707071, + "loss": 1.281, + "theoretical_loss": 3.4678114168176695, + "tokens_seen": 1758855168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750505050505051, + "loss": 1.392, + "theoretical_loss": 3.467800369475462, + "tokens_seen": 1758920704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047503030303030304, + "loss": 1.2578, + "theoretical_loss": 3.46778932266011, + "tokens_seen": 1758986240 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475010101010101, + "loss": 1.4468, + "theoretical_loss": 3.467778276371568, + "tokens_seen": 1759051776 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474989898989899, + "loss": 1.3945, + "theoretical_loss": 3.467767230609791, + "tokens_seen": 1759117312 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474969696969697, + "loss": 1.166, + "theoretical_loss": 3.4677561853747356, + "tokens_seen": 1759182848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047494949494949497, + "loss": 1.5393, + "theoretical_loss": 3.4677451406663558, + "tokens_seen": 1759248384 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047492929292929293, + "loss": 1.6032, + "theoretical_loss": 3.467734096484607, + "tokens_seen": 1759313920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749090909090909, + "loss": 1.4508, + "theoretical_loss": 3.467723052829445, + "tokens_seen": 1759379456 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047488888888888895, + "loss": 1.5765, + "theoretical_loss": 3.4677120097008247, + "tokens_seen": 1759444992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748686868686869, + "loss": 1.3183, + "theoretical_loss": 3.4677009670987013, + "tokens_seen": 1759510528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047484848484848487, + "loss": 1.3187, + "theoretical_loss": 3.4676899250230306, + "tokens_seen": 1759576064 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748282828282828, + "loss": 1.3925, + "theoretical_loss": 3.4676788834737677, + "tokens_seen": 1759641600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748080808080808, + "loss": 1.4587, + "theoretical_loss": 3.467667842450868, + "tokens_seen": 1759707136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047478787878787884, + "loss": 1.4543, + "theoretical_loss": 3.467656801954286, + "tokens_seen": 1759772672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747676767676768, + "loss": 1.3954, + "theoretical_loss": 3.4676457619839773, + "tokens_seen": 1759838208 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.3027859330177307, + "objective/train/avg_token_score": 0.026031043380498886, + "objective/train/docs_used": 996601, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0335179567337036, + "objective/train/original_loss": 4.650774955749512, + "objective/train/theoretical_loss": 3.4676347225398985, + "objective/train/tokens_used": 118762976, + "objective/train/unlikelihood_loss": 0.07695260643959045, + "theoretical_loss": 3.4676347225398985, + "tokens_seen": 1759903744 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047474747474747476, + "loss": 1.1674, + "theoretical_loss": 3.4676347225398985, + "tokens_seen": 1759903744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747272727272727, + "loss": 1.283, + "theoretical_loss": 3.467623683622003, + "tokens_seen": 1759969280 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047470707070707067, + "loss": 1.4078, + "theoretical_loss": 3.467612645230248, + "tokens_seen": 1760034816 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047468686868686874, + "loss": 1.3631, + "theoretical_loss": 3.4676016073645872, + "tokens_seen": 1760100352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746666666666667, + "loss": 1.4091, + "theoretical_loss": 3.467590570024977, + "tokens_seen": 1760165888 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047464646464646465, + "loss": 1.4418, + "theoretical_loss": 3.4675795332113726, + "tokens_seen": 1760231424 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746262626262626, + "loss": 1.3409, + "theoretical_loss": 3.467568496923729, + "tokens_seen": 1760296960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746060606060606, + "loss": 1.518, + "theoretical_loss": 3.4675574611620013, + "tokens_seen": 1760362496 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047458585858585863, + "loss": 1.4428, + "theoretical_loss": 3.4675464259261455, + "tokens_seen": 1760428032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745656565656566, + "loss": 1.3315, + "theoretical_loss": 3.467535391216116, + "tokens_seen": 1760493568 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047454545454545454, + "loss": 1.5314, + "theoretical_loss": 3.46752435703187, + "tokens_seen": 1760559104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745252525252525, + "loss": 1.5786, + "theoretical_loss": 3.4675133233733613, + "tokens_seen": 1760624640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745050505050505, + "loss": 1.5052, + "theoretical_loss": 3.467502290240546, + "tokens_seen": 1760690176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744848484848485, + "loss": 1.5072, + "theoretical_loss": 3.4674912576333785, + "tokens_seen": 1760755712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744646464646465, + "loss": 1.3799, + "theoretical_loss": 3.4674802255518156, + "tokens_seen": 1760821248 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047444444444444444, + "loss": 1.359, + "theoretical_loss": 3.467469193995812, + "tokens_seen": 1760886784 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047442424242424245, + "loss": 1.5932, + "theoretical_loss": 3.4674581629653227, + "tokens_seen": 1760952320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744040404040404, + "loss": 1.2436, + "theoretical_loss": 3.467447132460304, + "tokens_seen": 1761017856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743838383838384, + "loss": 1.2935, + "theoretical_loss": 3.467436102480711, + "tokens_seen": 1761083392 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047436363636363637, + "loss": 1.6615, + "theoretical_loss": 3.4674250730264986, + "tokens_seen": 1761148928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047434343434343433, + "loss": 1.4143, + "theoretical_loss": 3.4674140440976227, + "tokens_seen": 1761214464 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047432323232323234, + "loss": 1.5773, + "theoretical_loss": 3.467403015694039, + "tokens_seen": 1761280000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743030303030303, + "loss": 1.4823, + "theoretical_loss": 3.467391987815702, + "tokens_seen": 1761345536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742828282828283, + "loss": 1.282, + "theoretical_loss": 3.4673809604625676, + "tokens_seen": 1761411072 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047426262626262626, + "loss": 1.5621, + "theoretical_loss": 3.4673699336345916, + "tokens_seen": 1761476608 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.4690249264240265, + "objective/train/avg_token_score": 0.015736954286694527, + "objective/train/docs_used": 997243, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6199918985366821, + "objective/train/original_loss": 3.9470674991607666, + "objective/train/theoretical_loss": 3.4673589073317297, + "objective/train/tokens_used": 120401376, + "objective/train/unlikelihood_loss": 0.07392548024654388, + "theoretical_loss": 3.4673589073317297, + "tokens_seen": 1761542144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742424242424243, + "loss": 1.4835, + "theoretical_loss": 3.4673589073317297, + "tokens_seen": 1761542144 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047422222222222223, + "loss": 1.4607, + "theoretical_loss": 3.4673478815539363, + "tokens_seen": 1761607680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742020202020202, + "loss": 1.655, + "theoretical_loss": 3.4673368563011677, + "tokens_seen": 1761673216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741818181818182, + "loss": 1.4306, + "theoretical_loss": 3.467325831573379, + "tokens_seen": 1761738752 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047416161616161616, + "loss": 1.4634, + "theoretical_loss": 3.467314807370526, + "tokens_seen": 1761804288 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047414141414141417, + "loss": 1.4165, + "theoretical_loss": 3.4673037836925635, + "tokens_seen": 1761869824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741212121212121, + "loss": 1.4309, + "theoretical_loss": 3.467292760539448, + "tokens_seen": 1761935360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741010101010101, + "loss": 1.3544, + "theoretical_loss": 3.467281737911134, + "tokens_seen": 1762000896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004740808080808081, + "loss": 1.4597, + "theoretical_loss": 3.4672707158075777, + "tokens_seen": 1762066432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004740606060606061, + "loss": 1.5272, + "theoretical_loss": 3.467259694228734, + "tokens_seen": 1762131968 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047404040404040406, + "loss": 1.3645, + "theoretical_loss": 3.467248673174559, + "tokens_seen": 1762197504 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474020202020202, + "loss": 1.2698, + "theoretical_loss": 3.467237652645008, + "tokens_seen": 1762263040 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474, + "loss": 1.1518, + "theoretical_loss": 3.4672266326400365, + "tokens_seen": 1762328576 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473979797979798, + "loss": 1.446, + "theoretical_loss": 3.4672156131595995, + "tokens_seen": 1762394112 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473959595959596, + "loss": 1.4405, + "theoretical_loss": 3.4672045942036536, + "tokens_seen": 1762459648 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047393939393939395, + "loss": 1.2889, + "theoretical_loss": 3.4671935757721535, + "tokens_seen": 1762525184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739191919191919, + "loss": 1.389, + "theoretical_loss": 3.467182557865055, + "tokens_seen": 1762590720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047389898989898987, + "loss": 1.4165, + "theoretical_loss": 3.4671715404823136, + "tokens_seen": 1762656256 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047387878787878793, + "loss": 1.5323, + "theoretical_loss": 3.4671605236238854, + "tokens_seen": 1762721792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738585858585859, + "loss": 1.4547, + "theoretical_loss": 3.467149507289725, + "tokens_seen": 1762787328 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047383838383838385, + "loss": 1.3971, + "theoretical_loss": 3.4671384914797887, + "tokens_seen": 1762852864 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738181818181818, + "loss": 1.4061, + "theoretical_loss": 3.4671274761940314, + "tokens_seen": 1762918400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047379797979797976, + "loss": 1.2906, + "theoretical_loss": 3.4671164614324095, + "tokens_seen": 1762983936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737777777777778, + "loss": 1.4725, + "theoretical_loss": 3.4671054471948777, + "tokens_seen": 1763049472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737575757575758, + "loss": 1.2618, + "theoretical_loss": 3.467094433481392, + "tokens_seen": 1763115008 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.3298509418964386, + "objective/train/avg_token_score": 0.013608359731733799, + "objective/train/docs_used": 998400, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.140276312828064, + "objective/train/original_loss": 4.645390033721924, + "objective/train/theoretical_loss": 3.4670834202919085, + "objective/train/tokens_used": 122039776, + "objective/train/unlikelihood_loss": 0.06240946799516678, + "theoretical_loss": 3.4670834202919085, + "tokens_seen": 1763180544 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047373737373737374, + "loss": 1.3174, + "theoretical_loss": 3.4670834202919085, + "tokens_seen": 1763180544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737171717171717, + "loss": 1.6851, + "theoretical_loss": 3.4670724076263815, + "tokens_seen": 1763246080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736969696969697, + "loss": 1.3992, + "theoretical_loss": 3.4670613954847678, + "tokens_seen": 1763311616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736767676767677, + "loss": 1.482, + "theoretical_loss": 3.467050383867023, + "tokens_seen": 1763377152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736565656565657, + "loss": 1.4294, + "theoretical_loss": 3.467039372773102, + "tokens_seen": 1763442688 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047363636363636363, + "loss": 1.4823, + "theoretical_loss": 3.467028362202961, + "tokens_seen": 1763508224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736161616161616, + "loss": 1.4347, + "theoretical_loss": 3.467017352156555, + "tokens_seen": 1763573760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735959595959596, + "loss": 1.5324, + "theoretical_loss": 3.46700634263384, + "tokens_seen": 1763639296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735757575757576, + "loss": 1.2494, + "theoretical_loss": 3.466995333634772, + "tokens_seen": 1763704832 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047355555555555557, + "loss": 1.3624, + "theoretical_loss": 3.4669843251593058, + "tokens_seen": 1763770368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735353535353535, + "loss": 1.3935, + "theoretical_loss": 3.4669733172073975, + "tokens_seen": 1763835904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047351515151515154, + "loss": 1.4901, + "theoretical_loss": 3.466962309779003, + "tokens_seen": 1763901440 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734949494949495, + "loss": 1.3198, + "theoretical_loss": 3.4669513028740777, + "tokens_seen": 1763966976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734747474747475, + "loss": 1.2222, + "theoretical_loss": 3.466940296492577, + "tokens_seen": 1764032512 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047345454545454546, + "loss": 1.4412, + "theoretical_loss": 3.466929290634457, + "tokens_seen": 1764098048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734343434343434, + "loss": 1.2884, + "theoretical_loss": 3.466918285299673, + "tokens_seen": 1764163584 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047341414141414143, + "loss": 1.6086, + "theoretical_loss": 3.466907280488181, + "tokens_seen": 1764229120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733939393939394, + "loss": 1.3749, + "theoretical_loss": 3.4668962761999365, + "tokens_seen": 1764294656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733737373737374, + "loss": 1.2535, + "theoretical_loss": 3.4668852724348955, + "tokens_seen": 1764360192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047335353535353535, + "loss": 1.3333, + "theoretical_loss": 3.4668742691930126, + "tokens_seen": 1764425728 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047333333333333336, + "loss": 1.2971, + "theoretical_loss": 3.4668632664742454, + "tokens_seen": 1764491264 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733131313131313, + "loss": 1.3483, + "theoretical_loss": 3.4668522642785478, + "tokens_seen": 1764556800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732929292929293, + "loss": 1.2886, + "theoretical_loss": 3.466841262605876, + "tokens_seen": 1764622336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732727272727273, + "loss": 1.4083, + "theoretical_loss": 3.466830261456187, + "tokens_seen": 1764687872 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047325252525252525, + "loss": 1.4522, + "theoretical_loss": 3.4668192608294346, + "tokens_seen": 1764753408 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.30015885829925537, + "objective/train/avg_token_score": 0.025842411443591118, + "objective/train/docs_used": 998902, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9383648633956909, + "objective/train/original_loss": 4.241371154785156, + "objective/train/theoretical_loss": 3.4668082607255757, + "objective/train/tokens_used": 123678176, + "objective/train/unlikelihood_loss": 0.10302804410457611, + "theoretical_loss": 3.4668082607255757, + "tokens_seen": 1764818944 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047323232323232326, + "loss": 1.4569, + "theoretical_loss": 3.4668082607255757, + "tokens_seen": 1764818944 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732121212121212, + "loss": 1.4089, + "theoretical_loss": 3.4667972611445657, + "tokens_seen": 1764884480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731919191919192, + "loss": 1.4655, + "theoretical_loss": 3.46678626208636, + "tokens_seen": 1764950016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731717171717172, + "loss": 1.4081, + "theoretical_loss": 3.466775263550915, + "tokens_seen": 1765015552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731515151515152, + "loss": 1.5019, + "theoretical_loss": 3.466764265538186, + "tokens_seen": 1765081088 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047313131313131315, + "loss": 1.4618, + "theoretical_loss": 3.466753268048129, + "tokens_seen": 1765146624 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731111111111111, + "loss": 1.2713, + "theoretical_loss": 3.4667422710806997, + "tokens_seen": 1765212160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730909090909091, + "loss": 1.2932, + "theoretical_loss": 3.466731274635854, + "tokens_seen": 1765277696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730707070707071, + "loss": 1.6351, + "theoretical_loss": 3.466720278713547, + "tokens_seen": 1765343232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730505050505051, + "loss": 1.5274, + "theoretical_loss": 3.466709283313735, + "tokens_seen": 1765408768 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047303030303030304, + "loss": 1.4793, + "theoretical_loss": 3.4666982884363735, + "tokens_seen": 1765474304 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473010101010101, + "loss": 1.4963, + "theoretical_loss": 3.466687294081419, + "tokens_seen": 1765539840 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472989898989899, + "loss": 1.3953, + "theoretical_loss": 3.466676300248827, + "tokens_seen": 1765605376 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472969696969697, + "loss": 1.3848, + "theoretical_loss": 3.4666653069385527, + "tokens_seen": 1765670912 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472949494949495, + "loss": 1.3043, + "theoretical_loss": 3.466654314150552, + "tokens_seen": 1765736448 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047292929292929293, + "loss": 1.3659, + "theoretical_loss": 3.4666433218847814, + "tokens_seen": 1765801984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729090909090909, + "loss": 1.2315, + "theoretical_loss": 3.466632330141196, + "tokens_seen": 1765867520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728888888888889, + "loss": 1.412, + "theoretical_loss": 3.4666213389197518, + "tokens_seen": 1765933056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728686868686869, + "loss": 1.527, + "theoretical_loss": 3.4666103482204056, + "tokens_seen": 1765998592 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047284848484848487, + "loss": 1.1406, + "theoretical_loss": 3.466599358043111, + "tokens_seen": 1766064128 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047282828282828283, + "loss": 1.4347, + "theoretical_loss": 3.4665883683878262, + "tokens_seen": 1766129664 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728080808080808, + "loss": 1.2191, + "theoretical_loss": 3.466577379254506, + "tokens_seen": 1766195200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727878787878788, + "loss": 1.3334, + "theoretical_loss": 3.4665663906431057, + "tokens_seen": 1766260736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727676767676768, + "loss": 1.3408, + "theoretical_loss": 3.466555402553582, + "tokens_seen": 1766326272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047274747474747476, + "loss": 1.424, + "theoretical_loss": 3.466544414985891, + "tokens_seen": 1766391808 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.5058650970458984, + "objective/train/avg_token_score": 0.020609397441148758, + "objective/train/docs_used": 999561, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6941487789154053, + "objective/train/original_loss": 3.954047918319702, + "objective/train/theoretical_loss": 3.4665334279399875, + "objective/train/tokens_used": 125316576, + "objective/train/unlikelihood_loss": 0.04431372508406639, + "theoretical_loss": 3.4665334279399875, + "tokens_seen": 1766457344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727272727272727, + "loss": 1.5081, + "theoretical_loss": 3.4665334279399875, + "tokens_seen": 1766457344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727070707070707, + "loss": 1.2911, + "theoretical_loss": 3.4665224414158278, + "tokens_seen": 1766522880 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047268686868686874, + "loss": 1.4671, + "theoretical_loss": 3.466511455413368, + "tokens_seen": 1766588416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726666666666667, + "loss": 1.3627, + "theoretical_loss": 3.466500469932564, + "tokens_seen": 1766653952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047264646464646466, + "loss": 1.6863, + "theoretical_loss": 3.466489484973371, + "tokens_seen": 1766719488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726262626262626, + "loss": 1.3115, + "theoretical_loss": 3.4664785005357457, + "tokens_seen": 1766785024 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047260606060606057, + "loss": 1.4088, + "theoretical_loss": 3.4664675166196437, + "tokens_seen": 1766850560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047258585858585864, + "loss": 1.2904, + "theoretical_loss": 3.4664565332250215, + "tokens_seen": 1766916096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725656565656566, + "loss": 1.5834, + "theoretical_loss": 3.4664455503518337, + "tokens_seen": 1766981632 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047254545454545455, + "loss": 1.5353, + "theoretical_loss": 3.466434568000037, + "tokens_seen": 1767047168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725252525252525, + "loss": 1.2129, + "theoretical_loss": 3.4664235861695873, + "tokens_seen": 1767112704 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725050505050505, + "loss": 1.4923, + "theoretical_loss": 3.46641260486044, + "tokens_seen": 1767178240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047248484848484853, + "loss": 1.6744, + "theoretical_loss": 3.466401624072552, + "tokens_seen": 1767243776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724646464646465, + "loss": 1.1128, + "theoretical_loss": 3.4663906438058785, + "tokens_seen": 1767309312 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047244444444444444, + "loss": 1.5299, + "theoretical_loss": 3.4663796640603755, + "tokens_seen": 1767374848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724242424242424, + "loss": 1.4135, + "theoretical_loss": 3.466368684835999, + "tokens_seen": 1767440384 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724040404040404, + "loss": 1.3394, + "theoretical_loss": 3.466357706132705, + "tokens_seen": 1767505920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723838383838384, + "loss": 1.3015, + "theoretical_loss": 3.46634672795045, + "tokens_seen": 1767571456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723636363636364, + "loss": 1.3472, + "theoretical_loss": 3.466335750289189, + "tokens_seen": 1767636992 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047234343434343433, + "loss": 1.2866, + "theoretical_loss": 3.466324773148878, + "tokens_seen": 1767702528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047232323232323235, + "loss": 1.6653, + "theoretical_loss": 3.466313796529474, + "tokens_seen": 1767768064 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723030303030303, + "loss": 1.5335, + "theoretical_loss": 3.466302820430932, + "tokens_seen": 1767833600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004722828282828283, + "loss": 1.5068, + "theoretical_loss": 3.466291844853208, + "tokens_seen": 1767899136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047226262626262627, + "loss": 1.3592, + "theoretical_loss": 3.4662808697962584, + "tokens_seen": 1767964672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004722424242424242, + "loss": 1.4486, + "theoretical_loss": 3.466269895260039, + "tokens_seen": 1768030208 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.3491568863391876, + "objective/train/avg_token_score": 0.024049675092101097, + "objective/train/docs_used": 1000589, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.332099437713623, + "objective/train/original_loss": 4.583323001861572, + "objective/train/theoretical_loss": 3.466258921244506, + "objective/train/tokens_used": 126954976, + "objective/train/unlikelihood_loss": 0.07067796587944031, + "theoretical_loss": 3.466258921244506, + "tokens_seen": 1768095744 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047222222222222224, + "loss": 1.5076, + "theoretical_loss": 3.466258921244506, + "tokens_seen": 1768095744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004722020202020202, + "loss": 1.2966, + "theoretical_loss": 3.466247947749615, + "tokens_seen": 1768161280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721818181818182, + "loss": 1.2774, + "theoretical_loss": 3.466236974775322, + "tokens_seen": 1768226816 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047216161616161616, + "loss": 1.267, + "theoretical_loss": 3.4662260023215836, + "tokens_seen": 1768292352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721414141414142, + "loss": 1.362, + "theoretical_loss": 3.466215030388356, + "tokens_seen": 1768357888 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047212121212121213, + "loss": 1.427, + "theoretical_loss": 3.4662040589755936, + "tokens_seen": 1768423424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721010101010101, + "loss": 1.2644, + "theoretical_loss": 3.466193088083254, + "tokens_seen": 1768488960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720808080808081, + "loss": 1.5104, + "theoretical_loss": 3.466182117711293, + "tokens_seen": 1768554496 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047206060606060606, + "loss": 1.1877, + "theoretical_loss": 3.466171147859666, + "tokens_seen": 1768620032 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047204040404040407, + "loss": 1.3425, + "theoretical_loss": 3.46616017852833, + "tokens_seen": 1768685568 + }, + { + "epoch": 0.07, + "learning_rate": 0.000472020202020202, + "loss": 1.1985, + "theoretical_loss": 3.4661492097172397, + "tokens_seen": 1768751104 + }, + { + "epoch": 0.07, + "learning_rate": 0.000472, + "loss": 1.4138, + "theoretical_loss": 3.466138241426352, + "tokens_seen": 1768816640 + }, + { + "epoch": 0.07, + "learning_rate": 0.000471979797979798, + "loss": 1.3324, + "theoretical_loss": 3.466127273655623, + "tokens_seen": 1768882176 + }, + { + "epoch": 0.07, + "learning_rate": 0.000471959595959596, + "loss": 1.4765, + "theoretical_loss": 3.466116306405009, + "tokens_seen": 1768947712 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047193939393939396, + "loss": 1.2163, + "theoretical_loss": 3.466105339674465, + "tokens_seen": 1769013248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719191919191919, + "loss": 1.319, + "theoretical_loss": 3.466094373463948, + "tokens_seen": 1769078784 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718989898989899, + "loss": 1.3871, + "theoretical_loss": 3.4660834077734144, + "tokens_seen": 1769144320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718787878787879, + "loss": 1.3804, + "theoretical_loss": 3.466072442602819, + "tokens_seen": 1769209856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718585858585859, + "loss": 1.3414, + "theoretical_loss": 3.4660614779521186, + "tokens_seen": 1769275392 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047183838383838385, + "loss": 1.391, + "theoretical_loss": 3.4660505138212696, + "tokens_seen": 1769340928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718181818181818, + "loss": 1.4082, + "theoretical_loss": 3.466039550210228, + "tokens_seen": 1769406464 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047179797979797977, + "loss": 1.4775, + "theoretical_loss": 3.4660285871189496, + "tokens_seen": 1769472000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047177777777777783, + "loss": 1.4404, + "theoretical_loss": 3.4660176245473906, + "tokens_seen": 1769537536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717575757575758, + "loss": 1.4915, + "theoretical_loss": 3.4660066624955066, + "tokens_seen": 1769603072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047173737373737374, + "loss": 1.7011, + "theoretical_loss": 3.465995700963255, + "tokens_seen": 1769668608 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.48918622732162476, + "objective/train/avg_token_score": 0.019492249935865402, + "objective/train/docs_used": 1001288, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7274926900863647, + "objective/train/original_loss": 3.8678648471832275, + "objective/train/theoretical_loss": 3.4659847399505903, + "objective/train/tokens_used": 128593376, + "objective/train/unlikelihood_loss": 0.081143319606781, + "theoretical_loss": 3.4659847399505903, + "tokens_seen": 1769734144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717171717171717, + "loss": 1.3668, + "theoretical_loss": 3.4659847399505903, + "tokens_seen": 1769734144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047169696969696966, + "loss": 1.3583, + "theoretical_loss": 3.46597377945747, + "tokens_seen": 1769799680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716767676767677, + "loss": 1.3333, + "theoretical_loss": 3.46596281948385, + "tokens_seen": 1769865216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716565656565657, + "loss": 1.3922, + "theoretical_loss": 3.465951860029686, + "tokens_seen": 1769930752 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047163636363636364, + "loss": 1.519, + "theoretical_loss": 3.4659409010949345, + "tokens_seen": 1769996288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716161616161616, + "loss": 1.4696, + "theoretical_loss": 3.4659299426795513, + "tokens_seen": 1770061824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715959595959596, + "loss": 1.3336, + "theoretical_loss": 3.4659189847834924, + "tokens_seen": 1770127360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715757575757576, + "loss": 1.5364, + "theoretical_loss": 3.465908027406715, + "tokens_seen": 1770192896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715555555555556, + "loss": 1.3424, + "theoretical_loss": 3.465897070549174, + "tokens_seen": 1770258432 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047153535353535353, + "loss": 1.3519, + "theoretical_loss": 3.4658861142108264, + "tokens_seen": 1770323968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715151515151515, + "loss": 1.1967, + "theoretical_loss": 3.4658751583916283, + "tokens_seen": 1770389504 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047149494949494955, + "loss": 1.2003, + "theoretical_loss": 3.4658642030915354, + "tokens_seen": 1770455040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714747474747475, + "loss": 1.1804, + "theoretical_loss": 3.4658532483105042, + "tokens_seen": 1770520576 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047145454545454547, + "loss": 1.4514, + "theoretical_loss": 3.4658422940484908, + "tokens_seen": 1770586112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714343434343434, + "loss": 1.2616, + "theoretical_loss": 3.465831340305452, + "tokens_seen": 1770651648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714141414141414, + "loss": 1.1625, + "theoretical_loss": 3.4658203870813433, + "tokens_seen": 1770717184 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047139393939393944, + "loss": 1.7548, + "theoretical_loss": 3.465809434376121, + "tokens_seen": 1770782720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713737373737374, + "loss": 1.2719, + "theoretical_loss": 3.465798482189742, + "tokens_seen": 1770848256 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047135353535353536, + "loss": 1.3652, + "theoretical_loss": 3.465787530522161, + "tokens_seen": 1770913792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713333333333333, + "loss": 1.4807, + "theoretical_loss": 3.4657765793733355, + "tokens_seen": 1770979328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713131313131313, + "loss": 1.4566, + "theoretical_loss": 3.4657656287432212, + "tokens_seen": 1771044864 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047129292929292934, + "loss": 1.31, + "theoretical_loss": 3.465754678631775, + "tokens_seen": 1771110400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712727272727273, + "loss": 1.6366, + "theoretical_loss": 3.4657437290389526, + "tokens_seen": 1771175936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047125252525252525, + "loss": 1.1361, + "theoretical_loss": 3.4657327799647106, + "tokens_seen": 1771241472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712323232323232, + "loss": 1.3109, + "theoretical_loss": 3.4657218314090046, + "tokens_seen": 1771307008 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.5297531485557556, + "objective/train/avg_token_score": 0.0018574605928733945, + "objective/train/docs_used": 1002447, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.670203447341919, + "objective/train/original_loss": 3.6806411743164062, + "objective/train/theoretical_loss": 3.4657108833717913, + "objective/train/tokens_used": 130231776, + "objective/train/unlikelihood_loss": 0.10642728209495544, + "theoretical_loss": 3.4657108833717913, + "tokens_seen": 1771372544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712121212121212, + "loss": 1.7502, + "theoretical_loss": 3.4657108833717913, + "tokens_seen": 1771372544 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047119191919191923, + "loss": 1.4412, + "theoretical_loss": 3.465699935853027, + "tokens_seen": 1771438080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711717171717172, + "loss": 1.2746, + "theoretical_loss": 3.465688988852668, + "tokens_seen": 1771503616 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047115151515151514, + "loss": 1.5396, + "theoretical_loss": 3.46567804237067, + "tokens_seen": 1771569152 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047113131313131316, + "loss": 1.2664, + "theoretical_loss": 3.4656670964069907, + "tokens_seen": 1771634688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711111111111111, + "loss": 1.6049, + "theoretical_loss": 3.4656561509615846, + "tokens_seen": 1771700224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710909090909091, + "loss": 1.5435, + "theoretical_loss": 3.4656452060344085, + "tokens_seen": 1771765760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710707070707071, + "loss": 1.2803, + "theoretical_loss": 3.4656342616254197, + "tokens_seen": 1771831296 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047105050505050504, + "loss": 1.6564, + "theoretical_loss": 3.4656233177345737, + "tokens_seen": 1771896832 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047103030303030305, + "loss": 1.6557, + "theoretical_loss": 3.465612374361827, + "tokens_seen": 1771962368 + }, + { + "epoch": 0.07, + "learning_rate": 0.000471010101010101, + "loss": 1.2512, + "theoretical_loss": 3.4656014315071357, + "tokens_seen": 1772027904 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470989898989899, + "loss": 1.3483, + "theoretical_loss": 3.4655904891704563, + "tokens_seen": 1772093440 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047096969696969697, + "loss": 1.4258, + "theoretical_loss": 3.4655795473517452, + "tokens_seen": 1772158976 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470949494949495, + "loss": 1.414, + "theoretical_loss": 3.465568606050958, + "tokens_seen": 1772224512 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047092929292929294, + "loss": 1.2494, + "theoretical_loss": 3.465557665268052, + "tokens_seen": 1772290048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709090909090909, + "loss": 1.2409, + "theoretical_loss": 3.4655467250029837, + "tokens_seen": 1772355584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708888888888889, + "loss": 1.2723, + "theoretical_loss": 3.4655357852557085, + "tokens_seen": 1772421120 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047086868686868687, + "loss": 1.353, + "theoretical_loss": 3.4655248460261827, + "tokens_seen": 1772486656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708484848484849, + "loss": 1.7172, + "theoretical_loss": 3.4655139073143637, + "tokens_seen": 1772552192 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047082828282828283, + "loss": 1.4032, + "theoretical_loss": 3.465502969120207, + "tokens_seen": 1772617728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708080808080808, + "loss": 1.4746, + "theoretical_loss": 3.4654920314436697, + "tokens_seen": 1772683264 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707878787878788, + "loss": 1.3164, + "theoretical_loss": 3.465481094284707, + "tokens_seen": 1772748800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707676767676768, + "loss": 1.6484, + "theoretical_loss": 3.465470157643277, + "tokens_seen": 1772814336 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047074747474747477, + "loss": 1.4666, + "theoretical_loss": 3.4654592215193345, + "tokens_seen": 1772879872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707272727272727, + "loss": 1.616, + "theoretical_loss": 3.465448285912836, + "tokens_seen": 1772945408 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.42247068881988525, + "objective/train/avg_token_score": 0.04283915087580681, + "objective/train/docs_used": 1002969, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3265541791915894, + "objective/train/original_loss": 4.510785102844238, + "objective/train/theoretical_loss": 3.465437350823739, + "objective/train/tokens_used": 131870176, + "objective/train/unlikelihood_loss": 0.05501260235905647, + "theoretical_loss": 3.465437350823739, + "tokens_seen": 1773010944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707070707070707, + "loss": 1.5543, + "theoretical_loss": 3.465437350823739, + "tokens_seen": 1773010944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706868686868687, + "loss": 1.4789, + "theoretical_loss": 3.465426416251999, + "tokens_seen": 1773076480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706666666666667, + "loss": 1.3916, + "theoretical_loss": 3.4654154821975727, + "tokens_seen": 1773142016 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047064646464646466, + "loss": 1.5454, + "theoretical_loss": 3.465404548660416, + "tokens_seen": 1773207552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706262626262626, + "loss": 1.4936, + "theoretical_loss": 3.4653936156404868, + "tokens_seen": 1773273088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706060606060606, + "loss": 1.6112, + "theoretical_loss": 3.4653826831377397, + "tokens_seen": 1773338624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047058585858585864, + "loss": 1.4224, + "theoretical_loss": 3.465371751152132, + "tokens_seen": 1773404160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705656565656566, + "loss": 1.3528, + "theoretical_loss": 3.46536081968362, + "tokens_seen": 1773469696 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047054545454545455, + "loss": 1.5744, + "theoretical_loss": 3.4653498887321605, + "tokens_seen": 1773535232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705252525252525, + "loss": 1.6217, + "theoretical_loss": 3.465338958297709, + "tokens_seen": 1773600768 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047050505050505047, + "loss": 1.3906, + "theoretical_loss": 3.465328028380223, + "tokens_seen": 1773666304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047048484848484853, + "loss": 1.5699, + "theoretical_loss": 3.465317098979659, + "tokens_seen": 1773731840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704646464646465, + "loss": 1.4396, + "theoretical_loss": 3.465306170095972, + "tokens_seen": 1773797376 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047044444444444445, + "loss": 1.2902, + "theoretical_loss": 3.46529524172912, + "tokens_seen": 1773862912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704242424242424, + "loss": 1.526, + "theoretical_loss": 3.4652843138790583, + "tokens_seen": 1773928448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704040404040404, + "loss": 1.373, + "theoretical_loss": 3.4652733865457446, + "tokens_seen": 1773993984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703838383838384, + "loss": 1.4229, + "theoretical_loss": 3.465262459729135, + "tokens_seen": 1774059520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703636363636364, + "loss": 1.3665, + "theoretical_loss": 3.4652515334291847, + "tokens_seen": 1774125056 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047034343434343434, + "loss": 1.4426, + "theoretical_loss": 3.4652406076458515, + "tokens_seen": 1774190592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703232323232323, + "loss": 1.308, + "theoretical_loss": 3.465229682379092, + "tokens_seen": 1774256128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703030303030303, + "loss": 1.2964, + "theoretical_loss": 3.465218757628862, + "tokens_seen": 1774321664 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702828282828283, + "loss": 1.3539, + "theoretical_loss": 3.4652078333951186, + "tokens_seen": 1774387200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702626262626263, + "loss": 1.4111, + "theoretical_loss": 3.4651969096778177, + "tokens_seen": 1774452736 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047024242424242423, + "loss": 1.2421, + "theoretical_loss": 3.465185986476916, + "tokens_seen": 1774518272 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047022222222222224, + "loss": 1.3518, + "theoretical_loss": 3.465175063792371, + "tokens_seen": 1774583808 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.2966764271259308, + "objective/train/avg_token_score": 0.02469574473798275, + "objective/train/docs_used": 1004180, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0019471645355225, + "objective/train/original_loss": 4.505530834197998, + "objective/train/theoretical_loss": 3.465164141624137, + "objective/train/tokens_used": 133508576, + "objective/train/unlikelihood_loss": 0.06781195849180222, + "theoretical_loss": 3.465164141624137, + "tokens_seen": 1774649344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702020202020202, + "loss": 1.4618, + "theoretical_loss": 3.465164141624137, + "tokens_seen": 1774649344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701818181818182, + "loss": 1.6281, + "theoretical_loss": 3.465153219972173, + "tokens_seen": 1774714880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047016161616161617, + "loss": 1.3271, + "theoretical_loss": 3.4651422988364335, + "tokens_seen": 1774780416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701414141414141, + "loss": 1.3957, + "theoretical_loss": 3.4651313782168764, + "tokens_seen": 1774845952 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047012121212121214, + "loss": 1.5541, + "theoretical_loss": 3.465120458113458, + "tokens_seen": 1774911488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701010101010101, + "loss": 1.435, + "theoretical_loss": 3.4651095385261343, + "tokens_seen": 1774977024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700808080808081, + "loss": 1.6075, + "theoretical_loss": 3.4650986194548623, + "tokens_seen": 1775042560 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047006060606060606, + "loss": 1.3927, + "theoretical_loss": 3.4650877008995984, + "tokens_seen": 1775108096 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047004040404040407, + "loss": 1.2209, + "theoretical_loss": 3.4650767828602995, + "tokens_seen": 1775173632 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047002020202020203, + "loss": 1.3741, + "theoretical_loss": 3.4650658653369213, + "tokens_seen": 1775239168 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047, + "loss": 1.46, + "theoretical_loss": 3.4650549483294215, + "tokens_seen": 1775304704 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469979797979798, + "loss": 1.4385, + "theoretical_loss": 3.4650440318377562, + "tokens_seen": 1775370240 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046995959595959595, + "loss": 1.3606, + "theoretical_loss": 3.465033115861882, + "tokens_seen": 1775435776 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046993939393939396, + "loss": 1.1657, + "theoretical_loss": 3.465022200401755, + "tokens_seen": 1775501312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699191919191919, + "loss": 1.6816, + "theoretical_loss": 3.465011285457332, + "tokens_seen": 1775566848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698989898989899, + "loss": 1.3001, + "theoretical_loss": 3.4650003710285704, + "tokens_seen": 1775632384 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698787878787879, + "loss": 1.1577, + "theoretical_loss": 3.4649894571154265, + "tokens_seen": 1775697920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698585858585859, + "loss": 1.3568, + "theoretical_loss": 3.464978543717856, + "tokens_seen": 1775763456 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046983838383838386, + "loss": 1.614, + "theoretical_loss": 3.464967630835816, + "tokens_seen": 1775828992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698181818181818, + "loss": 1.2982, + "theoretical_loss": 3.464956718469264, + "tokens_seen": 1775894528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697979797979798, + "loss": 1.5625, + "theoretical_loss": 3.4649458066181555, + "tokens_seen": 1775960064 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697777777777778, + "loss": 1.6927, + "theoretical_loss": 3.4649348952824477, + "tokens_seen": 1776025600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697575757575758, + "loss": 1.4541, + "theoretical_loss": 3.4649239844620974, + "tokens_seen": 1776091136 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046973737373737375, + "loss": 1.4913, + "theoretical_loss": 3.46491307415706, + "tokens_seen": 1776156672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697171717171717, + "loss": 1.5123, + "theoretical_loss": 3.464902164367294, + "tokens_seen": 1776222208 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.2969208061695099, + "objective/train/avg_token_score": 0.01136105228215456, + "objective/train/docs_used": 1004852, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8510345816612244, + "objective/train/original_loss": 3.909015417098999, + "objective/train/theoretical_loss": 3.4648912550927546, + "objective/train/tokens_used": 135146976, + "objective/train/unlikelihood_loss": 0.0887172669172287, + "theoretical_loss": 3.4648912550927546, + "tokens_seen": 1776287744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696969696969697, + "loss": 1.363, + "theoretical_loss": 3.4648912550927546, + "tokens_seen": 1776287744 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046967676767676773, + "loss": 1.4334, + "theoretical_loss": 3.4648803463333993, + "tokens_seen": 1776353280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696565656565657, + "loss": 1.4213, + "theoretical_loss": 3.4648694380891847, + "tokens_seen": 1776418816 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046963636363636364, + "loss": 1.2654, + "theoretical_loss": 3.4648585303600665, + "tokens_seen": 1776484352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696161616161616, + "loss": 1.3323, + "theoretical_loss": 3.464847623146003, + "tokens_seen": 1776549888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695959595959596, + "loss": 1.4284, + "theoretical_loss": 3.464836716446949, + "tokens_seen": 1776615424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695757575757576, + "loss": 1.2791, + "theoretical_loss": 3.464825810262863, + "tokens_seen": 1776680960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695555555555556, + "loss": 1.1452, + "theoretical_loss": 3.4648149045937005, + "tokens_seen": 1776746496 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046953535353535354, + "loss": 1.5327, + "theoretical_loss": 3.464803999439418, + "tokens_seen": 1776812032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695151515151515, + "loss": 1.4731, + "theoretical_loss": 3.4647930947999743, + "tokens_seen": 1776877568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694949494949495, + "loss": 1.4519, + "theoretical_loss": 3.464782190675323, + "tokens_seen": 1776943104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694747474747475, + "loss": 1.4544, + "theoretical_loss": 3.4647712870654237, + "tokens_seen": 1777008640 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046945454545454547, + "loss": 1.46, + "theoretical_loss": 3.464760383970231, + "tokens_seen": 1777074176 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046943434343434343, + "loss": 1.3681, + "theoretical_loss": 3.4647494813897026, + "tokens_seen": 1777139712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694141414141414, + "loss": 1.4295, + "theoretical_loss": 3.464738579323795, + "tokens_seen": 1777205248 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046939393939393945, + "loss": 1.5011, + "theoretical_loss": 3.4647276777724647, + "tokens_seen": 1777270784 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693737373737374, + "loss": 1.3782, + "theoretical_loss": 3.4647167767356692, + "tokens_seen": 1777336320 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046935353535353536, + "loss": 1.5472, + "theoretical_loss": 3.4647058762133645, + "tokens_seen": 1777401856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693333333333333, + "loss": 1.2642, + "theoretical_loss": 3.4646949762055077, + "tokens_seen": 1777467392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693131313131313, + "loss": 1.292, + "theoretical_loss": 3.4646840767120555, + "tokens_seen": 1777532928 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046929292929292934, + "loss": 1.5226, + "theoretical_loss": 3.4646731777329647, + "tokens_seen": 1777598464 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692727272727273, + "loss": 1.4523, + "theoretical_loss": 3.4646622792681923, + "tokens_seen": 1777664000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046925252525252526, + "loss": 1.3278, + "theoretical_loss": 3.4646513813176942, + "tokens_seen": 1777729536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692323232323232, + "loss": 1.6332, + "theoretical_loss": 3.464640483881428, + "tokens_seen": 1777795072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692121212121212, + "loss": 1.388, + "theoretical_loss": 3.46462958695935, + "tokens_seen": 1777860608 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.34634652733802795, + "objective/train/avg_token_score": 0.013553972356021404, + "objective/train/docs_used": 1006253, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2541776895523071, + "objective/train/original_loss": 5.052961826324463, + "objective/train/theoretical_loss": 3.4646186905514176, + "objective/train/tokens_used": 136785376, + "objective/train/unlikelihood_loss": 0.05251467227935791, + "theoretical_loss": 3.4646186905514176, + "tokens_seen": 1777926144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046919191919191924, + "loss": 1.3604, + "theoretical_loss": 3.4646186905514176, + "tokens_seen": 1777926144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691717171717172, + "loss": 1.3728, + "theoretical_loss": 3.464607794657587, + "tokens_seen": 1777991680 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046915151515151515, + "loss": 1.4799, + "theoretical_loss": 3.464596899277815, + "tokens_seen": 1778057216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691313131313131, + "loss": 1.4162, + "theoretical_loss": 3.464586004412059, + "tokens_seen": 1778122752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691111111111111, + "loss": 1.364, + "theoretical_loss": 3.464575110060275, + "tokens_seen": 1778188288 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046909090909090913, + "loss": 1.2986, + "theoretical_loss": 3.4645642162224206, + "tokens_seen": 1778253824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690707070707071, + "loss": 1.3651, + "theoretical_loss": 3.4645533228984515, + "tokens_seen": 1778319360 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046905050505050504, + "loss": 1.2544, + "theoretical_loss": 3.464542430088326, + "tokens_seen": 1778384896 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046903030303030305, + "loss": 1.3842, + "theoretical_loss": 3.4645315377920003, + "tokens_seen": 1778450432 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469010101010101, + "loss": 1.4251, + "theoretical_loss": 3.46452064600943, + "tokens_seen": 1778515968 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468989898989899, + "loss": 1.5926, + "theoretical_loss": 3.4645097547405745, + "tokens_seen": 1778581504 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468969696969697, + "loss": 1.4541, + "theoretical_loss": 3.4644988639853884, + "tokens_seen": 1778647040 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046894949494949494, + "loss": 1.2371, + "theoretical_loss": 3.4644879737438297, + "tokens_seen": 1778712576 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046892929292929295, + "loss": 1.4936, + "theoretical_loss": 3.4644770840158543, + "tokens_seen": 1778778112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689090909090909, + "loss": 1.3229, + "theoretical_loss": 3.46446619480142, + "tokens_seen": 1778843648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688888888888889, + "loss": 1.2463, + "theoretical_loss": 3.4644553061004832, + "tokens_seen": 1778909184 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046886868686868687, + "loss": 1.4809, + "theoretical_loss": 3.4644444179130014, + "tokens_seen": 1778974720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688484848484849, + "loss": 1.2319, + "theoretical_loss": 3.46443353023893, + "tokens_seen": 1779040256 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046882828282828284, + "loss": 1.4577, + "theoretical_loss": 3.464422643078228, + "tokens_seen": 1779105792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688080808080808, + "loss": 1.4697, + "theoretical_loss": 3.4644117564308505, + "tokens_seen": 1779171328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687878787878788, + "loss": 1.4904, + "theoretical_loss": 3.464400870296755, + "tokens_seen": 1779236864 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046876767676767676, + "loss": 1.5906, + "theoretical_loss": 3.4643899846758983, + "tokens_seen": 1779302400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687474747474748, + "loss": 1.3874, + "theoretical_loss": 3.4643790995682373, + "tokens_seen": 1779367936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046872727272727273, + "loss": 1.6713, + "theoretical_loss": 3.46436821497373, + "tokens_seen": 1779433472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687070707070707, + "loss": 1.4633, + "theoretical_loss": 3.464357330892331, + "tokens_seen": 1779499008 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.4292522072792053, + "objective/train/avg_token_score": 0.015219959430396557, + "objective/train/docs_used": 1006973, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.337504506111145, + "objective/train/original_loss": 4.010739326477051, + "objective/train/theoretical_loss": 3.464346447323999, + "objective/train/tokens_used": 138423776, + "objective/train/unlikelihood_loss": 0.09005622565746307, + "theoretical_loss": 3.464346447323999, + "tokens_seen": 1779564544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686868686868687, + "loss": 1.3108, + "theoretical_loss": 3.464346447323999, + "tokens_seen": 1779564544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686666666666667, + "loss": 1.5185, + "theoretical_loss": 3.4643355642686906, + "tokens_seen": 1779630080 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046864646464646467, + "loss": 1.4462, + "theoretical_loss": 3.464324681726363, + "tokens_seen": 1779695616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686262626262626, + "loss": 1.2584, + "theoretical_loss": 3.4643137996969724, + "tokens_seen": 1779761152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686060606060606, + "loss": 1.5357, + "theoretical_loss": 3.4643029181804756, + "tokens_seen": 1779826688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685858585858586, + "loss": 1.4705, + "theoretical_loss": 3.464292037176831, + "tokens_seen": 1779892224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685656565656566, + "loss": 1.2145, + "theoretical_loss": 3.4642811566859937, + "tokens_seen": 1779957760 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046854545454545456, + "loss": 1.4919, + "theoretical_loss": 3.4642702767079214, + "tokens_seen": 1780023296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685252525252525, + "loss": 1.4393, + "theoretical_loss": 3.4642593972425715, + "tokens_seen": 1780088832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685050505050505, + "loss": 1.3427, + "theoretical_loss": 3.4642485182899008, + "tokens_seen": 1780154368 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046848484848484854, + "loss": 1.6959, + "theoretical_loss": 3.464237639849866, + "tokens_seen": 1780219904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684646464646465, + "loss": 1.2308, + "theoretical_loss": 3.464226761922424, + "tokens_seen": 1780285440 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046844444444444445, + "loss": 1.2129, + "theoretical_loss": 3.464215884507532, + "tokens_seen": 1780350976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684242424242424, + "loss": 1.4428, + "theoretical_loss": 3.464205007605147, + "tokens_seen": 1780416512 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046840404040404037, + "loss": 1.579, + "theoretical_loss": 3.464194131215226, + "tokens_seen": 1780482048 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046838383838383843, + "loss": 1.3781, + "theoretical_loss": 3.4641832553377254, + "tokens_seen": 1780547584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683636363636364, + "loss": 1.3164, + "theoretical_loss": 3.464172379972603, + "tokens_seen": 1780613120 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046834343434343435, + "loss": 1.5964, + "theoretical_loss": 3.4641615051198156, + "tokens_seen": 1780678656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683232323232323, + "loss": 1.4625, + "theoretical_loss": 3.4641506307793204, + "tokens_seen": 1780744192 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046830303030303026, + "loss": 1.3468, + "theoretical_loss": 3.4641397569510737, + "tokens_seen": 1780809728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682828282828283, + "loss": 1.4519, + "theoretical_loss": 3.464128883635033, + "tokens_seen": 1780875264 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682626262626263, + "loss": 1.3987, + "theoretical_loss": 3.4641180108311556, + "tokens_seen": 1780940800 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046824242424242424, + "loss": 1.3296, + "theoretical_loss": 3.4641071385393976, + "tokens_seen": 1781006336 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682222222222222, + "loss": 1.5257, + "theoretical_loss": 3.464096266759717, + "tokens_seen": 1781071872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682020202020202, + "loss": 1.5309, + "theoretical_loss": 3.4640853954920705, + "tokens_seen": 1781137408 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.3690738081932068, + "objective/train/avg_token_score": 0.019216934219002724, + "objective/train/docs_used": 1008497, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3794710636138916, + "objective/train/original_loss": 4.352183818817139, + "objective/train/theoretical_loss": 3.4640745247364153, + "objective/train/tokens_used": 140062176, + "objective/train/unlikelihood_loss": 0.05833863466978073, + "theoretical_loss": 3.4640745247364153, + "tokens_seen": 1781202944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681818181818182, + "loss": 1.5147, + "theoretical_loss": 3.4640745247364153, + "tokens_seen": 1781202944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681616161616162, + "loss": 1.2851, + "theoretical_loss": 3.4640636544927075, + "tokens_seen": 1781268480 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046814141414141413, + "loss": 1.5317, + "theoretical_loss": 3.4640527847609057, + "tokens_seen": 1781334016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681212121212121, + "loss": 1.3507, + "theoretical_loss": 3.4640419155409665, + "tokens_seen": 1781399552 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046810101010101015, + "loss": 1.3736, + "theoretical_loss": 3.4640310468328455, + "tokens_seen": 1781465088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680808080808081, + "loss": 1.4927, + "theoretical_loss": 3.4640201786365017, + "tokens_seen": 1781530624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046806060606060607, + "loss": 1.3966, + "theoretical_loss": 3.4640093109518917, + "tokens_seen": 1781596160 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468040404040404, + "loss": 1.316, + "theoretical_loss": 3.463998443778972, + "tokens_seen": 1781661696 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046802020202020203, + "loss": 1.2633, + "theoretical_loss": 3.4639875771177, + "tokens_seen": 1781727232 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046800000000000005, + "loss": 1.2474, + "theoretical_loss": 3.4639767109680326, + "tokens_seen": 1781792768 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467979797979798, + "loss": 1.5079, + "theoretical_loss": 3.463965845329927, + "tokens_seen": 1781858304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046795959595959596, + "loss": 1.3738, + "theoretical_loss": 3.46395498020334, + "tokens_seen": 1781923840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679393939393939, + "loss": 1.4382, + "theoretical_loss": 3.4639441155882302, + "tokens_seen": 1781989376 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046791919191919193, + "loss": 1.4504, + "theoretical_loss": 3.463933251484553, + "tokens_seen": 1782054912 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046789898989898994, + "loss": 1.5389, + "theoretical_loss": 3.463922387892266, + "tokens_seen": 1782120448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678787878787879, + "loss": 1.3819, + "theoretical_loss": 3.4639115248113264, + "tokens_seen": 1782185984 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046785858585858585, + "loss": 1.3085, + "theoretical_loss": 3.4639006622416915, + "tokens_seen": 1782251520 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046783838383838386, + "loss": 1.1682, + "theoretical_loss": 3.4638898001833183, + "tokens_seen": 1782317056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678181818181818, + "loss": 1.3635, + "theoretical_loss": 3.463878938636164, + "tokens_seen": 1782382592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046779797979797983, + "loss": 1.4278, + "theoretical_loss": 3.4638680776001856, + "tokens_seen": 1782448128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677777777777778, + "loss": 1.5485, + "theoretical_loss": 3.4638572170753408, + "tokens_seen": 1782513664 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046775757575757574, + "loss": 1.3842, + "theoretical_loss": 3.4638463570615854, + "tokens_seen": 1782579200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046773737373737376, + "loss": 1.3291, + "theoretical_loss": 3.463835497558878, + "tokens_seen": 1782644736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677171717171717, + "loss": 1.4354, + "theoretical_loss": 3.463824638567175, + "tokens_seen": 1782710272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676969696969697, + "loss": 1.3805, + "theoretical_loss": 3.4638137800864337, + "tokens_seen": 1782775808 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.41129031777381897, + "objective/train/avg_token_score": 0.034974388778209686, + "objective/train/docs_used": 1009251, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3251190185546875, + "objective/train/original_loss": 4.803771495819092, + "objective/train/theoretical_loss": 3.463802922116612, + "objective/train/tokens_used": 141700576, + "objective/train/unlikelihood_loss": 0.044761259108781815, + "theoretical_loss": 3.463802922116612, + "tokens_seen": 1782841344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676767676767677, + "loss": 1.3516, + "theoretical_loss": 3.463802922116612, + "tokens_seen": 1782841344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676565656565657, + "loss": 1.2101, + "theoretical_loss": 3.4637920646576656, + "tokens_seen": 1782906880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046763636363636365, + "loss": 1.4911, + "theoretical_loss": 3.463781207709553, + "tokens_seen": 1782972416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676161616161616, + "loss": 1.6088, + "theoretical_loss": 3.463770351272231, + "tokens_seen": 1783037952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675959595959596, + "loss": 1.377, + "theoretical_loss": 3.4637594953456565, + "tokens_seen": 1783103488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675757575757576, + "loss": 1.3993, + "theoretical_loss": 3.463748639929787, + "tokens_seen": 1783169024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675555555555556, + "loss": 1.4165, + "theoretical_loss": 3.4637377850245796, + "tokens_seen": 1783234560 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046753535353535354, + "loss": 1.3183, + "theoretical_loss": 3.4637269306299916, + "tokens_seen": 1783300096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675151515151515, + "loss": 1.436, + "theoretical_loss": 3.46371607674598, + "tokens_seen": 1783365632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674949494949495, + "loss": 1.3289, + "theoretical_loss": 3.463705223372502, + "tokens_seen": 1783431168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674747474747475, + "loss": 1.2814, + "theoretical_loss": 3.463694370509515, + "tokens_seen": 1783496704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674545454545455, + "loss": 1.4848, + "theoretical_loss": 3.4636835181569765, + "tokens_seen": 1783562240 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046743434343434343, + "loss": 1.3949, + "theoretical_loss": 3.4636726663148436, + "tokens_seen": 1783627776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674141414141414, + "loss": 1.2862, + "theoretical_loss": 3.463661814983073, + "tokens_seen": 1783693312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673939393939394, + "loss": 1.5041, + "theoretical_loss": 3.463650964161623, + "tokens_seen": 1783758848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673737373737374, + "loss": 1.3817, + "theoretical_loss": 3.4636401138504493, + "tokens_seen": 1783824384 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046735353535353537, + "loss": 1.6609, + "theoretical_loss": 3.4636292640495103, + "tokens_seen": 1783889920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673333333333333, + "loss": 1.2934, + "theoretical_loss": 3.4636184147587636, + "tokens_seen": 1783955456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673131313131313, + "loss": 1.3581, + "theoretical_loss": 3.4636075659781653, + "tokens_seen": 1784020992 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046729292929292935, + "loss": 1.3282, + "theoretical_loss": 3.4635967177076736, + "tokens_seen": 1784086528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672727272727273, + "loss": 1.4206, + "theoretical_loss": 3.463585869947245, + "tokens_seen": 1784152064 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046725252525252526, + "loss": 1.3386, + "theoretical_loss": 3.463575022696838, + "tokens_seen": 1784217600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672323232323232, + "loss": 1.4178, + "theoretical_loss": 3.4635641759564084, + "tokens_seen": 1784283136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672121212121212, + "loss": 1.427, + "theoretical_loss": 3.4635533297259142, + "tokens_seen": 1784348672 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046719191919191924, + "loss": 1.374, + "theoretical_loss": 3.463542484005313, + "tokens_seen": 1784414208 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.47012463212013245, + "objective/train/avg_token_score": 0.013481098227202892, + "objective/train/docs_used": 1010356, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.653877854347229, + "objective/train/original_loss": 4.1271233558654785, + "objective/train/theoretical_loss": 3.463531638794562, + "objective/train/tokens_used": 143338976, + "objective/train/unlikelihood_loss": 0.07333654910326004, + "theoretical_loss": 3.463531638794562, + "tokens_seen": 1784479744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004671717171717172, + "loss": 1.648, + "theoretical_loss": 3.463531638794562, + "tokens_seen": 1784479744 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046715151515151516, + "loss": 1.4924, + "theoretical_loss": 3.463520794093618, + "tokens_seen": 1784545280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671313131313131, + "loss": 1.4619, + "theoretical_loss": 3.4635099499024387, + "tokens_seen": 1784610816 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671111111111111, + "loss": 1.2438, + "theoretical_loss": 3.463499106220981, + "tokens_seen": 1784676352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046709090909090913, + "loss": 1.31, + "theoretical_loss": 3.463488263049203, + "tokens_seen": 1784741888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670707070707071, + "loss": 1.3666, + "theoretical_loss": 3.4634774203870617, + "tokens_seen": 1784807424 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046705050505050505, + "loss": 1.5564, + "theoretical_loss": 3.463466578234514, + "tokens_seen": 1784872960 + }, + { + "epoch": 0.08, + "learning_rate": 0.000467030303030303, + "loss": 1.513, + "theoretical_loss": 3.463455736591518, + "tokens_seen": 1784938496 + }, + { + "epoch": 0.08, + "learning_rate": 0.000467010101010101, + "loss": 1.4561, + "theoretical_loss": 3.4634448954580304, + "tokens_seen": 1785004032 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466989898989899, + "loss": 1.2265, + "theoretical_loss": 3.463434054834009, + "tokens_seen": 1785069568 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466969696969697, + "loss": 1.629, + "theoretical_loss": 3.463423214719411, + "tokens_seen": 1785135104 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046694949494949494, + "loss": 1.4004, + "theoretical_loss": 3.463412375114193, + "tokens_seen": 1785200640 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046692929292929295, + "loss": 1.5207, + "theoretical_loss": 3.4634015360183135, + "tokens_seen": 1785266176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669090909090909, + "loss": 1.6107, + "theoretical_loss": 3.4633906974317292, + "tokens_seen": 1785331712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668888888888889, + "loss": 1.0, + "theoretical_loss": 3.463379859354398, + "tokens_seen": 1785397248 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668686868686869, + "loss": 1.1273, + "theoretical_loss": 3.463369021786277, + "tokens_seen": 1785462784 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046684848484848483, + "loss": 1.2565, + "theoretical_loss": 3.463358184727323, + "tokens_seen": 1785528320 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046682828282828284, + "loss": 1.3508, + "theoretical_loss": 3.4633473481774946, + "tokens_seen": 1785593856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668080808080808, + "loss": 1.1729, + "theoretical_loss": 3.4633365121367485, + "tokens_seen": 1785659392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667878787878788, + "loss": 1.5044, + "theoretical_loss": 3.463325676605042, + "tokens_seen": 1785724928 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046676767676767677, + "loss": 1.5249, + "theoretical_loss": 3.4633148415823323, + "tokens_seen": 1785790464 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667474747474748, + "loss": 1.2512, + "theoretical_loss": 3.4633040070685777, + "tokens_seen": 1785856000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046672727272727274, + "loss": 1.6165, + "theoretical_loss": 3.463293173063735, + "tokens_seen": 1785921536 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667070707070707, + "loss": 1.3033, + "theoretical_loss": 3.463282339567762, + "tokens_seen": 1785987072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666868686868687, + "loss": 1.2939, + "theoretical_loss": 3.4632715065806154, + "tokens_seen": 1786052608 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.5376344323158264, + "objective/train/avg_token_score": 0.011802426539361477, + "objective/train/docs_used": 1010805, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8985878229141235, + "objective/train/original_loss": 3.866780996322632, + "objective/train/theoretical_loss": 3.463260674102253, + "objective/train/tokens_used": 144977376, + "objective/train/unlikelihood_loss": 0.08484289795160294, + "theoretical_loss": 3.463260674102253, + "tokens_seen": 1786118144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046666666666666666, + "loss": 1.5252, + "theoretical_loss": 3.463260674102253, + "tokens_seen": 1786118144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666464646464647, + "loss": 1.413, + "theoretical_loss": 3.4632498421326328, + "tokens_seen": 1786183680 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046662626262626263, + "loss": 1.3204, + "theoretical_loss": 3.4632390106717112, + "tokens_seen": 1786249216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666060606060606, + "loss": 1.5948, + "theoretical_loss": 3.4632281797194464, + "tokens_seen": 1786314752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665858585858586, + "loss": 1.5062, + "theoretical_loss": 3.463217349275796, + "tokens_seen": 1786380288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665656565656566, + "loss": 1.5023, + "theoretical_loss": 3.4632065193407167, + "tokens_seen": 1786445824 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046654545454545457, + "loss": 1.3644, + "theoretical_loss": 3.4631956899141665, + "tokens_seen": 1786511360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665252525252525, + "loss": 1.3282, + "theoretical_loss": 3.463184860996103, + "tokens_seen": 1786576896 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665050505050505, + "loss": 1.5257, + "theoretical_loss": 3.4631740325864833, + "tokens_seen": 1786642432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664848484848485, + "loss": 1.4772, + "theoretical_loss": 3.463163204685265, + "tokens_seen": 1786707968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664646464646465, + "loss": 1.2096, + "theoretical_loss": 3.4631523772924053, + "tokens_seen": 1786773504 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046644444444444446, + "loss": 1.583, + "theoretical_loss": 3.463141550407862, + "tokens_seen": 1786839040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664242424242424, + "loss": 1.3999, + "theoretical_loss": 3.4631307240315934, + "tokens_seen": 1786904576 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046640404040404037, + "loss": 1.535, + "theoretical_loss": 3.463119898163555, + "tokens_seen": 1786970112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663838383838384, + "loss": 1.4024, + "theoretical_loss": 3.463109072803706, + "tokens_seen": 1787035648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663636363636364, + "loss": 1.6384, + "theoretical_loss": 3.4630982479520034, + "tokens_seen": 1787101184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046634343434343435, + "loss": 1.4974, + "theoretical_loss": 3.4630874236084046, + "tokens_seen": 1787166720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663232323232323, + "loss": 1.3595, + "theoretical_loss": 3.4630765997728674, + "tokens_seen": 1787232256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663030303030303, + "loss": 1.2032, + "theoretical_loss": 3.463065776445349, + "tokens_seen": 1787297792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046628282828282833, + "loss": 1.6914, + "theoretical_loss": 3.463054953625807, + "tokens_seen": 1787363328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662626262626263, + "loss": 1.563, + "theoretical_loss": 3.4630441313141986, + "tokens_seen": 1787428864 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046624242424242424, + "loss": 1.6588, + "theoretical_loss": 3.4630333095104824, + "tokens_seen": 1787494400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662222222222222, + "loss": 1.3575, + "theoretical_loss": 3.463022488214615, + "tokens_seen": 1787559936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662020202020202, + "loss": 1.4825, + "theoretical_loss": 3.4630116674265548, + "tokens_seen": 1787625472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661818181818182, + "loss": 1.5337, + "theoretical_loss": 3.463000847146258, + "tokens_seen": 1787691008 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.27792033553123474, + "objective/train/avg_token_score": 0.018849870190024376, + "objective/train/docs_used": 1011887, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0499746799468994, + "objective/train/original_loss": 4.833069801330566, + "objective/train/theoretical_loss": 3.4629900273736833, + "objective/train/tokens_used": 146615776, + "objective/train/unlikelihood_loss": 0.06380809843540192, + "theoretical_loss": 3.4629900273736833, + "tokens_seen": 1787756544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661616161616162, + "loss": 1.4461, + "theoretical_loss": 3.4629900273736833, + "tokens_seen": 1787756544 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046614141414141414, + "loss": 1.2255, + "theoretical_loss": 3.462979208108788, + "tokens_seen": 1787822080 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661212121212121, + "loss": 1.6146, + "theoretical_loss": 3.462968389351529, + "tokens_seen": 1787887616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046610101010101016, + "loss": 1.5575, + "theoretical_loss": 3.4629575711018648, + "tokens_seen": 1787953152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004660808080808081, + "loss": 1.492, + "theoretical_loss": 3.462946753359753, + "tokens_seen": 1788018688 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046606060606060607, + "loss": 1.2284, + "theoretical_loss": 3.4629359361251506, + "tokens_seen": 1788084224 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046604040404040403, + "loss": 1.5239, + "theoretical_loss": 3.4629251193980153, + "tokens_seen": 1788149760 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466020202020202, + "loss": 1.4366, + "theoretical_loss": 3.462914303178305, + "tokens_seen": 1788215296 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046600000000000005, + "loss": 1.2893, + "theoretical_loss": 3.4629034874659768, + "tokens_seen": 1788280832 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465979797979798, + "loss": 1.5474, + "theoretical_loss": 3.4628926722609887, + "tokens_seen": 1788346368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046595959595959597, + "loss": 1.3654, + "theoretical_loss": 3.4628818575632985, + "tokens_seen": 1788411904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004659393939393939, + "loss": 1.4484, + "theoretical_loss": 3.4628710433728633, + "tokens_seen": 1788477440 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046591919191919193, + "loss": 1.4707, + "theoretical_loss": 3.462860229689641, + "tokens_seen": 1788542976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046589898989898994, + "loss": 1.5574, + "theoretical_loss": 3.4628494165135892, + "tokens_seen": 1788608512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658787878787879, + "loss": 1.3447, + "theoretical_loss": 3.4628386038446655, + "tokens_seen": 1788674048 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046585858585858586, + "loss": 1.4493, + "theoretical_loss": 3.4628277916828276, + "tokens_seen": 1788739584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658383838383838, + "loss": 1.5055, + "theoretical_loss": 3.462816980028033, + "tokens_seen": 1788805120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658181818181818, + "loss": 1.5176, + "theoretical_loss": 3.46280616888024, + "tokens_seen": 1788870656 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046579797979797984, + "loss": 1.3789, + "theoretical_loss": 3.462795358239405, + "tokens_seen": 1788936192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657777777777778, + "loss": 1.5674, + "theoretical_loss": 3.462784548105487, + "tokens_seen": 1789001728 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046575757575757575, + "loss": 1.3612, + "theoretical_loss": 3.4627737384784423, + "tokens_seen": 1789067264 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046573737373737376, + "loss": 1.4416, + "theoretical_loss": 3.46276292935823, + "tokens_seen": 1789132800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657171717171717, + "loss": 1.4044, + "theoretical_loss": 3.4627521207448066, + "tokens_seen": 1789198336 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046569696969696973, + "loss": 1.4033, + "theoretical_loss": 3.4627413126381303, + "tokens_seen": 1789263872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656767676767677, + "loss": 1.3574, + "theoretical_loss": 3.462730505038159, + "tokens_seen": 1789329408 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.47061339020729065, + "objective/train/avg_token_score": 0.010851036757230759, + "objective/train/docs_used": 1012559, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5306476354599, + "objective/train/original_loss": 4.064844608306885, + "objective/train/theoretical_loss": 3.46271969794485, + "objective/train/tokens_used": 148254176, + "objective/train/unlikelihood_loss": 0.07548725605010986, + "theoretical_loss": 3.46271969794485, + "tokens_seen": 1789394944 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046565656565656564, + "loss": 1.5801, + "theoretical_loss": 3.46271969794485, + "tokens_seen": 1789394944 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046563636363636365, + "loss": 1.2206, + "theoretical_loss": 3.4627088913581607, + "tokens_seen": 1789460480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656161616161616, + "loss": 1.3933, + "theoretical_loss": 3.4626980852780496, + "tokens_seen": 1789526016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655959595959596, + "loss": 1.4214, + "theoretical_loss": 3.4626872797044737, + "tokens_seen": 1789591552 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655757575757576, + "loss": 1.2427, + "theoretical_loss": 3.4626764746373913, + "tokens_seen": 1789657088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655555555555556, + "loss": 1.2924, + "theoretical_loss": 3.4626656700767597, + "tokens_seen": 1789722624 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046553535353535355, + "loss": 1.3831, + "theoretical_loss": 3.462654866022537, + "tokens_seen": 1789788160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655151515151515, + "loss": 1.4765, + "theoretical_loss": 3.4626440624746797, + "tokens_seen": 1789853696 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654949494949495, + "loss": 1.6304, + "theoretical_loss": 3.4626332594331473, + "tokens_seen": 1789919232 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046547474747474747, + "loss": 1.2434, + "theoretical_loss": 3.462622456897897, + "tokens_seen": 1789984768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654545454545455, + "loss": 1.2223, + "theoretical_loss": 3.4626116548688857, + "tokens_seen": 1790050304 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046543434343434344, + "loss": 1.5187, + "theoretical_loss": 3.462600853346072, + "tokens_seen": 1790115840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654141414141414, + "loss": 1.3788, + "theoretical_loss": 3.4625900523294133, + "tokens_seen": 1790181376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653939393939394, + "loss": 1.4454, + "theoretical_loss": 3.462579251818867, + "tokens_seen": 1790246912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653737373737374, + "loss": 1.3457, + "theoretical_loss": 3.4625684518143918, + "tokens_seen": 1790312448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653535353535354, + "loss": 1.4853, + "theoretical_loss": 3.4625576523159447, + "tokens_seen": 1790377984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046533333333333333, + "loss": 1.4036, + "theoretical_loss": 3.4625468533234836, + "tokens_seen": 1790443520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653131313131313, + "loss": 1.3071, + "theoretical_loss": 3.4625360548369666, + "tokens_seen": 1790509056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652929292929293, + "loss": 1.6316, + "theoretical_loss": 3.462525256856351, + "tokens_seen": 1790574592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652727272727273, + "loss": 1.2238, + "theoretical_loss": 3.462514459381595, + "tokens_seen": 1790640128 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046525252525252527, + "loss": 1.4982, + "theoretical_loss": 3.462503662412656, + "tokens_seen": 1790705664 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652323232323232, + "loss": 1.4581, + "theoretical_loss": 3.4624928659494927, + "tokens_seen": 1790771200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652121212121212, + "loss": 1.2027, + "theoretical_loss": 3.4624820699920615, + "tokens_seen": 1790836736 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046519191919191925, + "loss": 1.494, + "theoretical_loss": 3.462471274540321, + "tokens_seen": 1790902272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651717171717172, + "loss": 1.411, + "theoretical_loss": 3.4624604795942293, + "tokens_seen": 1790967808 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.4129398763179779, + "objective/train/avg_token_score": 0.012248078361153603, + "objective/train/docs_used": 1013169, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2744585275650024, + "objective/train/original_loss": 4.295300006866455, + "objective/train/theoretical_loss": 3.462449685153744, + "objective/train/tokens_used": 149892576, + "objective/train/unlikelihood_loss": 0.05737758427858353, + "theoretical_loss": 3.462449685153744, + "tokens_seen": 1791033344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046515151515151516, + "loss": 1.6501, + "theoretical_loss": 3.462449685153744, + "tokens_seen": 1791033344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651313131313131, + "loss": 1.6736, + "theoretical_loss": 3.462438891218822, + "tokens_seen": 1791098880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651111111111111, + "loss": 1.6972, + "theoretical_loss": 3.4624280977894224, + "tokens_seen": 1791164416 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046509090909090914, + "loss": 1.635, + "theoretical_loss": 3.462417304865502, + "tokens_seen": 1791229952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650707070707071, + "loss": 1.3991, + "theoretical_loss": 3.4624065124470196, + "tokens_seen": 1791295488 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046505050505050505, + "loss": 1.4326, + "theoretical_loss": 3.462395720533933, + "tokens_seen": 1791361024 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465030303030303, + "loss": 1.557, + "theoretical_loss": 3.462384929126199, + "tokens_seen": 1791426560 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046501010101010097, + "loss": 1.2343, + "theoretical_loss": 3.462374138223776, + "tokens_seen": 1791492096 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046498989898989903, + "loss": 1.5046, + "theoretical_loss": 3.4623633478266225, + "tokens_seen": 1791557632 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464969696969697, + "loss": 1.3209, + "theoretical_loss": 3.4623525579346954, + "tokens_seen": 1791623168 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046494949494949495, + "loss": 1.4695, + "theoretical_loss": 3.462341768547953, + "tokens_seen": 1791688704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649292929292929, + "loss": 1.2689, + "theoretical_loss": 3.462330979666353, + "tokens_seen": 1791754240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649090909090909, + "loss": 1.4558, + "theoretical_loss": 3.462320191289854, + "tokens_seen": 1791819776 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648888888888889, + "loss": 1.6189, + "theoretical_loss": 3.4623094034184128, + "tokens_seen": 1791885312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648686868686869, + "loss": 1.3666, + "theoretical_loss": 3.4622986160519877, + "tokens_seen": 1791950848 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046484848484848484, + "loss": 1.3568, + "theoretical_loss": 3.4622878291905366, + "tokens_seen": 1792016384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648282828282828, + "loss": 1.328, + "theoretical_loss": 3.4622770428340184, + "tokens_seen": 1792081920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648080808080808, + "loss": 1.4596, + "theoretical_loss": 3.4622662569823888, + "tokens_seen": 1792147456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647878787878788, + "loss": 1.5656, + "theoretical_loss": 3.4622554716356078, + "tokens_seen": 1792212992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647676767676768, + "loss": 1.5854, + "theoretical_loss": 3.462244686793632, + "tokens_seen": 1792278528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046474747474747473, + "loss": 1.4945, + "theoretical_loss": 3.46223390245642, + "tokens_seen": 1792344064 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046472727272727274, + "loss": 1.4186, + "theoretical_loss": 3.4622231186239296, + "tokens_seen": 1792409600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647070707070707, + "loss": 1.3961, + "theoretical_loss": 3.4622123352961185, + "tokens_seen": 1792475136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646868686868687, + "loss": 1.4866, + "theoretical_loss": 3.462201552472945, + "tokens_seen": 1792540672 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046466666666666667, + "loss": 1.4341, + "theoretical_loss": 3.4621907701543666, + "tokens_seen": 1792606208 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.4124511182308197, + "objective/train/avg_token_score": 0.007224504370242357, + "objective/train/docs_used": 1014342, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.416115403175354, + "objective/train/original_loss": 3.994509220123291, + "objective/train/theoretical_loss": 3.462179988340341, + "objective/train/tokens_used": 151530976, + "objective/train/unlikelihood_loss": 0.08687492460012436, + "theoretical_loss": 3.462179988340341, + "tokens_seen": 1792671744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646464646464646, + "loss": 1.4909, + "theoretical_loss": 3.462179988340341, + "tokens_seen": 1792671744 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046462626262626264, + "loss": 1.2383, + "theoretical_loss": 3.4621692070308274, + "tokens_seen": 1792737280 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046460606060606065, + "loss": 1.3036, + "theoretical_loss": 3.4621584262257823, + "tokens_seen": 1792802816 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645858585858586, + "loss": 1.3584, + "theoretical_loss": 3.4621476459251648, + "tokens_seen": 1792868352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046456565656565656, + "loss": 1.7118, + "theoretical_loss": 3.4621368661289322, + "tokens_seen": 1792933888 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046454545454545457, + "loss": 1.6058, + "theoretical_loss": 3.462126086837042, + "tokens_seen": 1792999424 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046452525252525253, + "loss": 1.2664, + "theoretical_loss": 3.4621153080494538, + "tokens_seen": 1793064960 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046450505050505054, + "loss": 1.3595, + "theoretical_loss": 3.462104529766124, + "tokens_seen": 1793130496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644848484848485, + "loss": 1.4667, + "theoretical_loss": 3.462093751987011, + "tokens_seen": 1793196032 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046446464646464645, + "loss": 1.4804, + "theoretical_loss": 3.4620829747120734, + "tokens_seen": 1793261568 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046444444444444446, + "loss": 1.7482, + "theoretical_loss": 3.4620721979412687, + "tokens_seen": 1793327104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644242424242424, + "loss": 1.3312, + "theoretical_loss": 3.462061421674554, + "tokens_seen": 1793392640 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046440404040404043, + "loss": 1.4292, + "theoretical_loss": 3.46205064591189, + "tokens_seen": 1793458176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643838383838384, + "loss": 1.4, + "theoretical_loss": 3.462039870653231, + "tokens_seen": 1793523712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643636363636364, + "loss": 1.3416, + "theoretical_loss": 3.4620290958985382, + "tokens_seen": 1793589248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046434343434343436, + "loss": 1.5423, + "theoretical_loss": 3.462018321647768, + "tokens_seen": 1793654784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643232323232323, + "loss": 1.3754, + "theoretical_loss": 3.462007547900879, + "tokens_seen": 1793720320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643030303030303, + "loss": 1.6072, + "theoretical_loss": 3.4619967746578286, + "tokens_seen": 1793785856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642828282828283, + "loss": 1.3222, + "theoretical_loss": 3.4619860019185755, + "tokens_seen": 1793851392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642626262626263, + "loss": 1.375, + "theoretical_loss": 3.4619752296830772, + "tokens_seen": 1793916928 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046424242424242425, + "loss": 1.3657, + "theoretical_loss": 3.461964457951292, + "tokens_seen": 1793982464 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642222222222222, + "loss": 1.4207, + "theoretical_loss": 3.461953686723178, + "tokens_seen": 1794048000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642020202020202, + "loss": 1.2671, + "theoretical_loss": 3.461942915998694, + "tokens_seen": 1794113536 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046418181818181823, + "loss": 1.4485, + "theoretical_loss": 3.4619321457777965, + "tokens_seen": 1794179072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641616161616162, + "loss": 1.4694, + "theoretical_loss": 3.4619213760604444, + "tokens_seen": 1794244608 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.4748900234699249, + "objective/train/avg_token_score": 0.019929148256778717, + "objective/train/docs_used": 1015045, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.42253839969635, + "objective/train/original_loss": 4.371203422546387, + "objective/train/theoretical_loss": 3.4619106068465957, + "objective/train/tokens_used": 153169376, + "objective/train/unlikelihood_loss": 0.07094332575798035, + "theoretical_loss": 3.4619106068465957, + "tokens_seen": 1794310144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046414141414141414, + "loss": 1.4498, + "theoretical_loss": 3.4619106068465957, + "tokens_seen": 1794310144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641212121212121, + "loss": 1.1417, + "theoretical_loss": 3.4618998381362087, + "tokens_seen": 1794375680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641010101010101, + "loss": 1.1921, + "theoretical_loss": 3.4618890699292413, + "tokens_seen": 1794441216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640808080808081, + "loss": 1.1941, + "theoretical_loss": 3.461878302225651, + "tokens_seen": 1794506752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640606060606061, + "loss": 1.4643, + "theoretical_loss": 3.4618675350253967, + "tokens_seen": 1794572288 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046404040404040403, + "loss": 1.4376, + "theoretical_loss": 3.4618567683284365, + "tokens_seen": 1794637824 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464020202020202, + "loss": 1.2493, + "theoretical_loss": 3.4618460021347275, + "tokens_seen": 1794703360 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046400000000000006, + "loss": 1.3667, + "theoretical_loss": 3.4618352364442293, + "tokens_seen": 1794768896 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463979797979798, + "loss": 1.4107, + "theoretical_loss": 3.4618244712568984, + "tokens_seen": 1794834432 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046395959595959597, + "loss": 1.3153, + "theoretical_loss": 3.4618137065726944, + "tokens_seen": 1794899968 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046393939393939393, + "loss": 1.3565, + "theoretical_loss": 3.4618029423915746, + "tokens_seen": 1794965504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639191919191919, + "loss": 1.4403, + "theoretical_loss": 3.461792178713497, + "tokens_seen": 1795031040 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046389898989898995, + "loss": 1.356, + "theoretical_loss": 3.46178141553842, + "tokens_seen": 1795096576 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638787878787879, + "loss": 1.1425, + "theoretical_loss": 3.461770652866302, + "tokens_seen": 1795162112 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046385858585858586, + "loss": 1.3359, + "theoretical_loss": 3.4617598906971008, + "tokens_seen": 1795227648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638383838383838, + "loss": 1.337, + "theoretical_loss": 3.4617491290307743, + "tokens_seen": 1795293184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046381818181818183, + "loss": 1.6086, + "theoretical_loss": 3.461738367867281, + "tokens_seen": 1795358720 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046379797979797984, + "loss": 1.4242, + "theoretical_loss": 3.461727607206579, + "tokens_seen": 1795424256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637777777777778, + "loss": 1.5684, + "theoretical_loss": 3.461716847048627, + "tokens_seen": 1795489792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046375757575757576, + "loss": 1.4422, + "theoretical_loss": 3.461706087393382, + "tokens_seen": 1795555328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637373737373737, + "loss": 1.6148, + "theoretical_loss": 3.461695328240803, + "tokens_seen": 1795620864 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637171717171717, + "loss": 1.0624, + "theoretical_loss": 3.461684569590848, + "tokens_seen": 1795686400 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046369696969696974, + "loss": 1.5246, + "theoretical_loss": 3.461673811443475, + "tokens_seen": 1795751936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636767676767677, + "loss": 1.4155, + "theoretical_loss": 3.4616630537986426, + "tokens_seen": 1795817472 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046365656565656565, + "loss": 1.3022, + "theoretical_loss": 3.4616522966563084, + "tokens_seen": 1795883008 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.38929617404937744, + "objective/train/avg_token_score": 0.010316792875528336, + "objective/train/docs_used": 1016473, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3159542083740234, + "objective/train/original_loss": 4.452950954437256, + "objective/train/theoretical_loss": 3.461641540016431, + "objective/train/tokens_used": 154807776, + "objective/train/unlikelihood_loss": 0.060373466461896896, + "theoretical_loss": 3.461641540016431, + "tokens_seen": 1795948544 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046363636363636366, + "loss": 1.411, + "theoretical_loss": 3.461641540016431, + "tokens_seen": 1795948544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636161616161616, + "loss": 1.3186, + "theoretical_loss": 3.4616307838789684, + "tokens_seen": 1796014080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046359595959595963, + "loss": 1.4337, + "theoretical_loss": 3.4616200282438783, + "tokens_seen": 1796079616 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635757575757576, + "loss": 1.3708, + "theoretical_loss": 3.4616092731111205, + "tokens_seen": 1796145152 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046355555555555554, + "loss": 1.41, + "theoretical_loss": 3.4615985184806517, + "tokens_seen": 1796210688 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046353535353535355, + "loss": 1.3782, + "theoretical_loss": 3.461587764352431, + "tokens_seen": 1796276224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635151515151515, + "loss": 1.3656, + "theoretical_loss": 3.4615770107264154, + "tokens_seen": 1796341760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634949494949495, + "loss": 1.314, + "theoretical_loss": 3.461566257602565, + "tokens_seen": 1796407296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634747474747475, + "loss": 1.2375, + "theoretical_loss": 3.4615555049808364, + "tokens_seen": 1796472832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634545454545455, + "loss": 1.5319, + "theoretical_loss": 3.4615447528611885, + "tokens_seen": 1796538368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046343434343434345, + "loss": 1.5685, + "theoretical_loss": 3.46153400124358, + "tokens_seen": 1796603904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634141414141414, + "loss": 1.321, + "theoretical_loss": 3.461523250127968, + "tokens_seen": 1796669440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633939393939394, + "loss": 1.2743, + "theoretical_loss": 3.4615124995143116, + "tokens_seen": 1796734976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046337373737373737, + "loss": 1.3477, + "theoretical_loss": 3.4615017494025686, + "tokens_seen": 1796800512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633535353535354, + "loss": 1.7626, + "theoretical_loss": 3.461490999792698, + "tokens_seen": 1796866048 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046333333333333334, + "loss": 1.7518, + "theoretical_loss": 3.461480250684657, + "tokens_seen": 1796931584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633131313131313, + "loss": 1.3579, + "theoretical_loss": 3.4614695020784048, + "tokens_seen": 1796997120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632929292929293, + "loss": 1.2737, + "theoretical_loss": 3.4614587539738992, + "tokens_seen": 1797062656 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632727272727273, + "loss": 1.5971, + "theoretical_loss": 3.4614480063710986, + "tokens_seen": 1797128192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632525252525253, + "loss": 1.3421, + "theoretical_loss": 3.461437259269961, + "tokens_seen": 1797193728 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046323232323232323, + "loss": 1.3835, + "theoretical_loss": 3.4614265126704455, + "tokens_seen": 1797259264 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632121212121212, + "loss": 1.3964, + "theoretical_loss": 3.46141576657251, + "tokens_seen": 1797324800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631919191919192, + "loss": 1.3049, + "theoretical_loss": 3.461405020976112, + "tokens_seen": 1797390336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631717171717172, + "loss": 1.5157, + "theoretical_loss": 3.4613942758812106, + "tokens_seen": 1797455872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046315151515151517, + "loss": 1.4886, + "theoretical_loss": 3.4613835312877645, + "tokens_seen": 1797521408 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.45002442598342896, + "objective/train/avg_token_score": 0.027033532038331032, + "objective/train/docs_used": 1017203, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.404165506362915, + "objective/train/original_loss": 4.197830677032471, + "objective/train/theoretical_loss": 3.461372787195731, + "objective/train/tokens_used": 156446176, + "objective/train/unlikelihood_loss": 0.05572577565908432, + "theoretical_loss": 3.461372787195731, + "tokens_seen": 1797586944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631313131313131, + "loss": 1.5094, + "theoretical_loss": 3.461372787195731, + "tokens_seen": 1797586944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631111111111111, + "loss": 1.0144, + "theoretical_loss": 3.461362043605069, + "tokens_seen": 1797652480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630909090909091, + "loss": 1.6355, + "theoretical_loss": 3.4613513005157373, + "tokens_seen": 1797718016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630707070707071, + "loss": 1.6244, + "theoretical_loss": 3.4613405579276932, + "tokens_seen": 1797783552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046305050505050506, + "loss": 1.4438, + "theoretical_loss": 3.4613298158408954, + "tokens_seen": 1797849088 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463030303030303, + "loss": 1.1163, + "theoretical_loss": 3.461319074255303, + "tokens_seen": 1797914624 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046301010101010097, + "loss": 1.3346, + "theoretical_loss": 3.461308333170873, + "tokens_seen": 1797980160 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046298989898989904, + "loss": 1.5162, + "theoretical_loss": 3.461297592587565, + "tokens_seen": 1798045696 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462969696969697, + "loss": 1.3135, + "theoretical_loss": 3.461286852505336, + "tokens_seen": 1798111232 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046294949494949495, + "loss": 1.5227, + "theoretical_loss": 3.4612761129241463, + "tokens_seen": 1798176768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629292929292929, + "loss": 1.4748, + "theoretical_loss": 3.4612653738439523, + "tokens_seen": 1798242304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629090909090909, + "loss": 1.2214, + "theoretical_loss": 3.4612546352647136, + "tokens_seen": 1798307840 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046288888888888893, + "loss": 1.1131, + "theoretical_loss": 3.461243897186388, + "tokens_seen": 1798373376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628686868686869, + "loss": 1.3627, + "theoretical_loss": 3.461233159608934, + "tokens_seen": 1798438912 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046284848484848484, + "loss": 1.5596, + "theoretical_loss": 3.4612224225323103, + "tokens_seen": 1798504448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628282828282828, + "loss": 1.3134, + "theoretical_loss": 3.461211685956475, + "tokens_seen": 1798569984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046280808080808087, + "loss": 1.3904, + "theoretical_loss": 3.4612009498813867, + "tokens_seen": 1798635520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627878787878788, + "loss": 1.3134, + "theoretical_loss": 3.4611902143070035, + "tokens_seen": 1798701056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627676767676768, + "loss": 1.2739, + "theoretical_loss": 3.4611794792332837, + "tokens_seen": 1798766592 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046274747474747474, + "loss": 1.2228, + "theoretical_loss": 3.4611687446601866, + "tokens_seen": 1798832128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627272727272727, + "loss": 1.2976, + "theoretical_loss": 3.4611580105876696, + "tokens_seen": 1798897664 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046270707070707076, + "loss": 1.3079, + "theoretical_loss": 3.4611472770156917, + "tokens_seen": 1798963200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626868686868687, + "loss": 1.3461, + "theoretical_loss": 3.4611365439442103, + "tokens_seen": 1799028736 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626666666666667, + "loss": 1.4401, + "theoretical_loss": 3.4611258113731855, + "tokens_seen": 1799094272 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046264646464646463, + "loss": 1.3903, + "theoretical_loss": 3.4611150793025747, + "tokens_seen": 1799159808 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.5161290168762207, + "objective/train/avg_token_score": 0.012231235392391682, + "objective/train/docs_used": 1018428, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5293020009994507, + "objective/train/original_loss": 3.6187143325805664, + "objective/train/theoretical_loss": 3.461104347732337, + "objective/train/tokens_used": 158084576, + "objective/train/unlikelihood_loss": 0.06525947153568268, + "theoretical_loss": 3.461104347732337, + "tokens_seen": 1799225344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046262626262626264, + "loss": 1.4842, + "theoretical_loss": 3.461104347732337, + "tokens_seen": 1799225344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046260606060606065, + "loss": 1.5779, + "theoretical_loss": 3.4610936166624295, + "tokens_seen": 1799290880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625858585858586, + "loss": 1.5093, + "theoretical_loss": 3.4610828860928122, + "tokens_seen": 1799356416 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046256565656565657, + "loss": 1.3926, + "theoretical_loss": 3.4610721560234428, + "tokens_seen": 1799421952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625454545454545, + "loss": 1.3863, + "theoretical_loss": 3.46106142645428, + "tokens_seen": 1799487488 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046252525252525253, + "loss": 1.3946, + "theoretical_loss": 3.4610506973852813, + "tokens_seen": 1799553024 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046250505050505054, + "loss": 1.5555, + "theoretical_loss": 3.4610399688164066, + "tokens_seen": 1799618560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624848484848485, + "loss": 1.5781, + "theoretical_loss": 3.4610292407476138, + "tokens_seen": 1799684096 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046246464646464646, + "loss": 1.3303, + "theoretical_loss": 3.4610185131788613, + "tokens_seen": 1799749632 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046244444444444447, + "loss": 1.4305, + "theoretical_loss": 3.461007786110108, + "tokens_seen": 1799815168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624242424242424, + "loss": 1.3423, + "theoretical_loss": 3.4609970595413113, + "tokens_seen": 1799880704 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046240404040404044, + "loss": 1.2689, + "theoretical_loss": 3.4609863334724307, + "tokens_seen": 1799946240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623838383838384, + "loss": 1.3048, + "theoretical_loss": 3.460975607903425, + "tokens_seen": 1800011776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046236363636363635, + "loss": 1.4274, + "theoretical_loss": 3.460964882834251, + "tokens_seen": 1800077312 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046234343434343436, + "loss": 1.4462, + "theoretical_loss": 3.460954158264869, + "tokens_seen": 1800142848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623232323232323, + "loss": 1.5049, + "theoretical_loss": 3.460943434195237, + "tokens_seen": 1800208384 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046230303030303033, + "loss": 1.5564, + "theoretical_loss": 3.460932710625313, + "tokens_seen": 1800273920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622828282828283, + "loss": 1.6183, + "theoretical_loss": 3.4609219875550563, + "tokens_seen": 1800339456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622626262626263, + "loss": 1.3066, + "theoretical_loss": 3.460911264984425, + "tokens_seen": 1800404992 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046224242424242425, + "loss": 1.4609, + "theoretical_loss": 3.460900542913377, + "tokens_seen": 1800470528 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622222222222222, + "loss": 1.0031, + "theoretical_loss": 3.460889821341872, + "tokens_seen": 1800536064 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622020202020202, + "loss": 1.5134, + "theoretical_loss": 3.4608791002698682, + "tokens_seen": 1800601600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621818181818182, + "loss": 1.3774, + "theoretical_loss": 3.460868379697324, + "tokens_seen": 1800667136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621616161616162, + "loss": 1.4704, + "theoretical_loss": 3.460857659624198, + "tokens_seen": 1800732672 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046214141414141415, + "loss": 1.6048, + "theoretical_loss": 3.4608469400504482, + "tokens_seen": 1800798208 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.4208821952342987, + "objective/train/avg_token_score": 0.024829581379890442, + "objective/train/docs_used": 1018922, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4344236850738525, + "objective/train/original_loss": 4.255611419677734, + "objective/train/theoretical_loss": 3.4608362209760335, + "objective/train/tokens_used": 159722976, + "objective/train/unlikelihood_loss": 0.07284124940633774, + "theoretical_loss": 3.4608362209760335, + "tokens_seen": 1800863744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004621212121212121, + "loss": 1.3354, + "theoretical_loss": 3.4608362209760335, + "tokens_seen": 1800863744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004621010101010101, + "loss": 1.73, + "theoretical_loss": 3.4608255024009136, + "tokens_seen": 1800929280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620808080808081, + "loss": 1.5529, + "theoretical_loss": 3.4608147843250454, + "tokens_seen": 1800994816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620606060606061, + "loss": 1.3241, + "theoretical_loss": 3.4608040667483886, + "tokens_seen": 1801060352 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046204040404040404, + "loss": 1.3847, + "theoretical_loss": 3.460793349670901, + "tokens_seen": 1801125888 + }, + { + "epoch": 0.09, + "learning_rate": 0.000462020202020202, + "loss": 1.603, + "theoretical_loss": 3.460782633092542, + "tokens_seen": 1801191424 + }, + { + "epoch": 0.09, + "learning_rate": 0.000462, + "loss": 1.4548, + "theoretical_loss": 3.4607719170132696, + "tokens_seen": 1801256960 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461979797979798, + "loss": 1.5245, + "theoretical_loss": 3.4607612014330424, + "tokens_seen": 1801322496 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461959595959596, + "loss": 1.2273, + "theoretical_loss": 3.4607504863518193, + "tokens_seen": 1801388032 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046193939393939393, + "loss": 1.6071, + "theoretical_loss": 3.460739771769559, + "tokens_seen": 1801453568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619191919191919, + "loss": 1.4026, + "theoretical_loss": 3.46072905768622, + "tokens_seen": 1801519104 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046189898989898996, + "loss": 1.4037, + "theoretical_loss": 3.4607183441017604, + "tokens_seen": 1801584640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618787878787879, + "loss": 1.3166, + "theoretical_loss": 3.460707631016139, + "tokens_seen": 1801650176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046185858585858587, + "loss": 1.3045, + "theoretical_loss": 3.4606969184293153, + "tokens_seen": 1801715712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618383838383838, + "loss": 1.7083, + "theoretical_loss": 3.460686206341247, + "tokens_seen": 1801781248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618181818181818, + "loss": 1.531, + "theoretical_loss": 3.4606754947518934, + "tokens_seen": 1801846784 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046179797979797985, + "loss": 1.3199, + "theoretical_loss": 3.4606647836612128, + "tokens_seen": 1801912320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617777777777778, + "loss": 1.3986, + "theoretical_loss": 3.460654073069163, + "tokens_seen": 1801977856 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046175757575757576, + "loss": 1.3911, + "theoretical_loss": 3.4606433629757043, + "tokens_seen": 1802043392 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617373737373737, + "loss": 1.279, + "theoretical_loss": 3.4606326533807943, + "tokens_seen": 1802108928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617171717171717, + "loss": 1.4534, + "theoretical_loss": 3.4606219442843917, + "tokens_seen": 1802174464 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046169696969696974, + "loss": 1.5236, + "theoretical_loss": 3.460611235686456, + "tokens_seen": 1802240000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616767676767677, + "loss": 1.5087, + "theoretical_loss": 3.4606005275869447, + "tokens_seen": 1802305536 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046165656565656565, + "loss": 1.4131, + "theoretical_loss": 3.460589819985817, + "tokens_seen": 1802371072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616363636363636, + "loss": 1.4507, + "theoretical_loss": 3.460579112883032, + "tokens_seen": 1802436608 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.33669355511665344, + "objective/train/avg_token_score": 0.009447429329156876, + "objective/train/docs_used": 1019593, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2540216445922852, + "objective/train/original_loss": 4.323973655700684, + "objective/train/theoretical_loss": 3.460568406278548, + "objective/train/tokens_used": 161361376, + "objective/train/unlikelihood_loss": 0.07911861687898636, + "theoretical_loss": 3.460568406278548, + "tokens_seen": 1802502144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616161616161616, + "loss": 1.3935, + "theoretical_loss": 3.460568406278548, + "tokens_seen": 1802502144 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046159595959595963, + "loss": 1.336, + "theoretical_loss": 3.4605577001723233, + "tokens_seen": 1802567680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615757575757576, + "loss": 1.4096, + "theoretical_loss": 3.460546994564317, + "tokens_seen": 1802633216 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046155555555555555, + "loss": 1.3497, + "theoretical_loss": 3.460536289454488, + "tokens_seen": 1802698752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615353535353535, + "loss": 1.5146, + "theoretical_loss": 3.460525584842795, + "tokens_seen": 1802764288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615151515151515, + "loss": 1.3389, + "theoretical_loss": 3.4605148807291966, + "tokens_seen": 1802829824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614949494949495, + "loss": 1.3034, + "theoretical_loss": 3.460504177113651, + "tokens_seen": 1802895360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614747474747475, + "loss": 1.542, + "theoretical_loss": 3.4604934739961175, + "tokens_seen": 1802960896 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046145454545454544, + "loss": 1.5322, + "theoretical_loss": 3.460482771376555, + "tokens_seen": 1803026432 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046143434343434345, + "loss": 1.4973, + "theoretical_loss": 3.460472069254922, + "tokens_seen": 1803091968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614141414141414, + "loss": 1.2724, + "theoretical_loss": 3.4604613676311766, + "tokens_seen": 1803157504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613939393939394, + "loss": 1.3662, + "theoretical_loss": 3.4604506665052783, + "tokens_seen": 1803223040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613737373737374, + "loss": 1.4956, + "theoretical_loss": 3.4604399658771863, + "tokens_seen": 1803288576 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046135353535353533, + "loss": 1.1918, + "theoretical_loss": 3.460429265746858, + "tokens_seen": 1803354112 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046133333333333334, + "loss": 1.2887, + "theoretical_loss": 3.460418566114253, + "tokens_seen": 1803419648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613131313131313, + "loss": 1.2515, + "theoretical_loss": 3.4604078669793297, + "tokens_seen": 1803485184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612929292929293, + "loss": 1.1434, + "theoretical_loss": 3.4603971683420474, + "tokens_seen": 1803550720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046127272727272727, + "loss": 1.5023, + "theoretical_loss": 3.4603864702023643, + "tokens_seen": 1803616256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612525252525253, + "loss": 1.495, + "theoretical_loss": 3.4603757725602398, + "tokens_seen": 1803681792 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046123232323232324, + "loss": 1.493, + "theoretical_loss": 3.4603650754156323, + "tokens_seen": 1803747328 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612121212121212, + "loss": 1.5071, + "theoretical_loss": 3.4603543787685, + "tokens_seen": 1803812864 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611919191919192, + "loss": 1.3662, + "theoretical_loss": 3.460343682618803, + "tokens_seen": 1803878400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046117171717171716, + "loss": 1.3852, + "theoretical_loss": 3.460332986966499, + "tokens_seen": 1803943936 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046115151515151517, + "loss": 1.3354, + "theoretical_loss": 3.4603222918115475, + "tokens_seen": 1804009472 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046113131313131313, + "loss": 1.2229, + "theoretical_loss": 3.4603115971539067, + "tokens_seen": 1804075008 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.36766862869262695, + "objective/train/avg_token_score": 0.01413616631180048, + "objective/train/docs_used": 1020148, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.183333396911621, + "objective/train/original_loss": 4.116270065307617, + "objective/train/theoretical_loss": 3.460300902993536, + "objective/train/tokens_used": 162999776, + "objective/train/unlikelihood_loss": 0.08400173485279083, + "theoretical_loss": 3.460300902993536, + "tokens_seen": 1804140544 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046111111111111114, + "loss": 1.4689, + "theoretical_loss": 3.460300902993536, + "tokens_seen": 1804140544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610909090909091, + "loss": 1.4244, + "theoretical_loss": 3.4602902093303936, + "tokens_seen": 1804206080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610707070707071, + "loss": 1.6157, + "theoretical_loss": 3.460279516164439, + "tokens_seen": 1804271616 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046105050505050506, + "loss": 1.3807, + "theoretical_loss": 3.460268823495631, + "tokens_seen": 1804337152 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461030303030303, + "loss": 1.3098, + "theoretical_loss": 3.4602581313239273, + "tokens_seen": 1804402688 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046101010101010103, + "loss": 1.4589, + "theoretical_loss": 3.460247439649288, + "tokens_seen": 1804468224 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460989898989899, + "loss": 1.2463, + "theoretical_loss": 3.4602367484716714, + "tokens_seen": 1804533760 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460969696969697, + "loss": 1.5439, + "theoretical_loss": 3.4602260577910364, + "tokens_seen": 1804599296 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046094949494949496, + "loss": 1.3021, + "theoretical_loss": 3.4602153676073417, + "tokens_seen": 1804664832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609292929292929, + "loss": 1.4273, + "theoretical_loss": 3.4602046779205464, + "tokens_seen": 1804730368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609090909090909, + "loss": 1.587, + "theoretical_loss": 3.4601939887306097, + "tokens_seen": 1804795904 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046088888888888894, + "loss": 1.2694, + "theoretical_loss": 3.4601833000374898, + "tokens_seen": 1804861440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608686868686869, + "loss": 1.3032, + "theoretical_loss": 3.4601726118411458, + "tokens_seen": 1804926976 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046084848484848485, + "loss": 1.4727, + "theoretical_loss": 3.460161924141537, + "tokens_seen": 1804992512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608282828282828, + "loss": 1.4011, + "theoretical_loss": 3.4601512369386214, + "tokens_seen": 1805058048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608080808080808, + "loss": 1.2257, + "theoretical_loss": 3.4601405502323583, + "tokens_seen": 1805123584 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046078787878787883, + "loss": 1.373, + "theoretical_loss": 3.460129864022707, + "tokens_seen": 1805189120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607676767676768, + "loss": 1.2235, + "theoretical_loss": 3.460119178309626, + "tokens_seen": 1805254656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046074747474747474, + "loss": 1.5574, + "theoretical_loss": 3.4601084930930743, + "tokens_seen": 1805320192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607272727272727, + "loss": 1.3258, + "theoretical_loss": 3.4600978083730105, + "tokens_seen": 1805385728 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046070707070707077, + "loss": 1.4703, + "theoretical_loss": 3.460087124149394, + "tokens_seen": 1805451264 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606868686868687, + "loss": 1.2853, + "theoretical_loss": 3.4600764404221835, + "tokens_seen": 1805516800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606666666666667, + "loss": 1.5211, + "theoretical_loss": 3.460065757191338, + "tokens_seen": 1805582336 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046064646464646464, + "loss": 1.5547, + "theoretical_loss": 3.460055074456816, + "tokens_seen": 1805647872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606262626262626, + "loss": 1.3416, + "theoretical_loss": 3.4600443922185766, + "tokens_seen": 1805713408 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.40169844031333923, + "objective/train/avg_token_score": 0.0685867890715599, + "objective/train/docs_used": 1021470, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3303582668304443, + "objective/train/original_loss": 4.088999271392822, + "objective/train/theoretical_loss": 3.4600337104765795, + "objective/train/tokens_used": 164638176, + "objective/train/unlikelihood_loss": 0.06166958063840866, + "theoretical_loss": 3.4600337104765795, + "tokens_seen": 1805778944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046060606060606066, + "loss": 1.422, + "theoretical_loss": 3.4600337104765795, + "tokens_seen": 1805778944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605858585858586, + "loss": 1.3202, + "theoretical_loss": 3.4600230292307828, + "tokens_seen": 1805844480 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046056565656565657, + "loss": 1.2661, + "theoretical_loss": 3.4600123484811456, + "tokens_seen": 1805910016 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046054545454545453, + "loss": 1.5477, + "theoretical_loss": 3.460001668227627, + "tokens_seen": 1805975552 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046052525252525254, + "loss": 1.2371, + "theoretical_loss": 3.459990988470186, + "tokens_seen": 1806041088 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046050505050505055, + "loss": 1.3883, + "theoretical_loss": 3.459980309208781, + "tokens_seen": 1806106624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604848484848485, + "loss": 1.4474, + "theoretical_loss": 3.459969630443372, + "tokens_seen": 1806172160 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046046464646464646, + "loss": 1.4572, + "theoretical_loss": 3.459958952173917, + "tokens_seen": 1806237696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604444444444444, + "loss": 1.3687, + "theoretical_loss": 3.459948274400375, + "tokens_seen": 1806303232 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046042424242424243, + "loss": 1.5883, + "theoretical_loss": 3.459937597122706, + "tokens_seen": 1806368768 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046040404040404044, + "loss": 1.113, + "theoretical_loss": 3.459926920340868, + "tokens_seen": 1806434304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603838383838384, + "loss": 1.3119, + "theoretical_loss": 3.4599162440548197, + "tokens_seen": 1806499840 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046036363636363636, + "loss": 1.1213, + "theoretical_loss": 3.4599055682645217, + "tokens_seen": 1806565376 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046034343434343437, + "loss": 1.5036, + "theoretical_loss": 3.459894892969931, + "tokens_seen": 1806630912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603232323232323, + "loss": 1.4614, + "theoretical_loss": 3.459884218171008, + "tokens_seen": 1806696448 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046030303030303034, + "loss": 1.2264, + "theoretical_loss": 3.4598735438677117, + "tokens_seen": 1806761984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602828282828283, + "loss": 1.2862, + "theoretical_loss": 3.4598628700600003, + "tokens_seen": 1806827520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046026262626262625, + "loss": 1.4969, + "theoretical_loss": 3.4598521967478337, + "tokens_seen": 1806893056 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046024242424242426, + "loss": 1.3143, + "theoretical_loss": 3.4598415239311695, + "tokens_seen": 1806958592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602222222222222, + "loss": 1.3717, + "theoretical_loss": 3.4598308516099685, + "tokens_seen": 1807024128 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046020202020202023, + "loss": 1.2754, + "theoretical_loss": 3.4598201797841885, + "tokens_seen": 1807089664 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601818181818182, + "loss": 1.3765, + "theoretical_loss": 3.459809508453789, + "tokens_seen": 1807155200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601616161616162, + "loss": 1.4214, + "theoretical_loss": 3.459798837618729, + "tokens_seen": 1807220736 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046014141414141415, + "loss": 1.2824, + "theoretical_loss": 3.4597881672789677, + "tokens_seen": 1807286272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601212121212121, + "loss": 1.2776, + "theoretical_loss": 3.4597774974344637, + "tokens_seen": 1807351808 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.5522360801696777, + "objective/train/avg_token_score": 0.005671007093042135, + "objective/train/docs_used": 1022105, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.969338059425354, + "objective/train/original_loss": 3.9141592979431152, + "objective/train/theoretical_loss": 3.4597668280851765, + "objective/train/tokens_used": 166276576, + "objective/train/unlikelihood_loss": 0.07195361703634262, + "theoretical_loss": 3.4597668280851765, + "tokens_seen": 1807417344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601010101010101, + "loss": 1.4951, + "theoretical_loss": 3.4597668280851765, + "tokens_seen": 1807417344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600808080808081, + "loss": 1.4208, + "theoretical_loss": 3.4597561592310644, + "tokens_seen": 1807482880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600606060606061, + "loss": 1.4108, + "theoretical_loss": 3.4597454908720877, + "tokens_seen": 1807548416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046004040404040405, + "loss": 1.5542, + "theoretical_loss": 3.459734823008205, + "tokens_seen": 1807613952 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460020202020202, + "loss": 1.2047, + "theoretical_loss": 3.4597241556393747, + "tokens_seen": 1807679488 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046, + "loss": 1.4753, + "theoretical_loss": 3.4597134887655567, + "tokens_seen": 1807745024 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459979797979798, + "loss": 1.4563, + "theoretical_loss": 3.459702822386709, + "tokens_seen": 1807810560 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459959595959596, + "loss": 1.4975, + "theoretical_loss": 3.4596921565027925, + "tokens_seen": 1807876096 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045993939393939394, + "loss": 1.5808, + "theoretical_loss": 3.4596814911137646, + "tokens_seen": 1807941632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599191919191919, + "loss": 1.3581, + "theoretical_loss": 3.459670826219585, + "tokens_seen": 1808007168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598989898989899, + "loss": 1.4058, + "theoretical_loss": 3.459660161820213, + "tokens_seen": 1808072704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598787878787879, + "loss": 1.4865, + "theoretical_loss": 3.459649497915608, + "tokens_seen": 1808138240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598585858585859, + "loss": 1.3384, + "theoretical_loss": 3.459638834505728, + "tokens_seen": 1808203776 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045983838383838383, + "loss": 1.2745, + "theoretical_loss": 3.4596281715905333, + "tokens_seen": 1808269312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598181818181818, + "loss": 1.2995, + "theoretical_loss": 3.4596175091699823, + "tokens_seen": 1808334848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597979797979798, + "loss": 1.2344, + "theoretical_loss": 3.459606847244034, + "tokens_seen": 1808400384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597777777777778, + "loss": 1.2524, + "theoretical_loss": 3.4595961858126483, + "tokens_seen": 1808465920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045975757575757577, + "loss": 1.3612, + "theoretical_loss": 3.459585524875784, + "tokens_seen": 1808531456 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597373737373737, + "loss": 1.3054, + "theoretical_loss": 3.4595748644334, + "tokens_seen": 1808596992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597171717171717, + "loss": 1.4338, + "theoretical_loss": 3.4595642044854555, + "tokens_seen": 1808662528 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045969696969696975, + "loss": 1.3573, + "theoretical_loss": 3.45955354503191, + "tokens_seen": 1808728064 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596767676767677, + "loss": 1.2754, + "theoretical_loss": 3.4595428860727218, + "tokens_seen": 1808793600 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045965656565656566, + "loss": 1.45, + "theoretical_loss": 3.4595322276078515, + "tokens_seen": 1808859136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596363636363636, + "loss": 1.3942, + "theoretical_loss": 3.459521569637257, + "tokens_seen": 1808924672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596161616161616, + "loss": 1.5969, + "theoretical_loss": 3.459510912160898, + "tokens_seen": 1808990208 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.4491080045700073, + "objective/train/avg_token_score": 0.009337084367871284, + "objective/train/docs_used": 1022607, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.514532446861267, + "objective/train/original_loss": 3.918682336807251, + "objective/train/theoretical_loss": 3.4595002551787335, + "objective/train/tokens_used": 167914976, + "objective/train/unlikelihood_loss": 0.08269507437944412, + "theoretical_loss": 3.4595002551787335, + "tokens_seen": 1809055744 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045959595959595964, + "loss": 1.38, + "theoretical_loss": 3.4595002551787335, + "tokens_seen": 1809055744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595757575757576, + "loss": 1.4447, + "theoretical_loss": 3.4594895986907233, + "tokens_seen": 1809121280 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045955555555555555, + "loss": 1.2231, + "theoretical_loss": 3.4594789426968253, + "tokens_seen": 1809186816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595353535353535, + "loss": 1.5118, + "theoretical_loss": 3.4594682871969997, + "tokens_seen": 1809252352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595151515151515, + "loss": 1.3982, + "theoretical_loss": 3.4594576321912056, + "tokens_seen": 1809317888 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045949494949494953, + "loss": 1.3473, + "theoretical_loss": 3.459446977679402, + "tokens_seen": 1809383424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594747474747475, + "loss": 1.4469, + "theoretical_loss": 3.4594363236615484, + "tokens_seen": 1809448960 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045945454545454545, + "loss": 1.1953, + "theoretical_loss": 3.4594256701376036, + "tokens_seen": 1809514496 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594343434343434, + "loss": 1.4947, + "theoretical_loss": 3.459415017107527, + "tokens_seen": 1809580032 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045941414141414147, + "loss": 1.1828, + "theoretical_loss": 3.4594043645712773, + "tokens_seen": 1809645568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593939393939394, + "loss": 1.5579, + "theoretical_loss": 3.459393712528815, + "tokens_seen": 1809711104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593737373737374, + "loss": 1.2093, + "theoretical_loss": 3.459383060980098, + "tokens_seen": 1809776640 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045935353535353534, + "loss": 1.3925, + "theoretical_loss": 3.459372409925087, + "tokens_seen": 1809842176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045933333333333335, + "loss": 1.5784, + "theoretical_loss": 3.4593617593637394, + "tokens_seen": 1809907712 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045931313131313136, + "loss": 1.4518, + "theoretical_loss": 3.4593511092960156, + "tokens_seen": 1809973248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592929292929293, + "loss": 1.2988, + "theoretical_loss": 3.459340459721875, + "tokens_seen": 1810038784 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592727272727273, + "loss": 1.4289, + "theoretical_loss": 3.459329810641276, + "tokens_seen": 1810104320 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045925252525252523, + "loss": 1.4281, + "theoretical_loss": 3.4593191620541783, + "tokens_seen": 1810169856 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045923232323232324, + "loss": 1.2985, + "theoretical_loss": 3.4593085139605417, + "tokens_seen": 1810235392 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045921212121212125, + "loss": 1.3204, + "theoretical_loss": 3.459297866360325, + "tokens_seen": 1810300928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004591919191919192, + "loss": 1.4494, + "theoretical_loss": 3.459287219253487, + "tokens_seen": 1810366464 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045917171717171717, + "loss": 1.5598, + "theoretical_loss": 3.459276572639988, + "tokens_seen": 1810432000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004591515151515152, + "loss": 1.3901, + "theoretical_loss": 3.459265926519786, + "tokens_seen": 1810497536 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045913131313131313, + "loss": 1.3048, + "theoretical_loss": 3.459255280892841, + "tokens_seen": 1810563072 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045911111111111115, + "loss": 1.4061, + "theoretical_loss": 3.459244635759113, + "tokens_seen": 1810628608 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.2624022364616394, + "objective/train/avg_token_score": 0.027153268456459045, + "objective/train/docs_used": 1024104, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0158097743988037, + "objective/train/original_loss": 5.203606128692627, + "objective/train/theoretical_loss": 3.459233991118561, + "objective/train/tokens_used": 169553376, + "objective/train/unlikelihood_loss": 0.043073952198028564, + "theoretical_loss": 3.459233991118561, + "tokens_seen": 1810694144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004590909090909091, + "loss": 1.26, + "theoretical_loss": 3.459233991118561, + "tokens_seen": 1810694144 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045907070707070706, + "loss": 1.2706, + "theoretical_loss": 3.4592233469711426, + "tokens_seen": 1810759680 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045905050505050507, + "loss": 1.0934, + "theoretical_loss": 3.459212703316819, + "tokens_seen": 1810825216 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045903030303030303, + "loss": 1.3351, + "theoretical_loss": 3.4592020601555493, + "tokens_seen": 1810890752 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045901010101010104, + "loss": 1.2675, + "theoretical_loss": 3.4591914174872915, + "tokens_seen": 1810956288 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458989898989899, + "loss": 1.3725, + "theoretical_loss": 3.4591807753120065, + "tokens_seen": 1811021824 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458969696969697, + "loss": 1.4893, + "theoretical_loss": 3.4591701336296534, + "tokens_seen": 1811087360 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045894949494949496, + "loss": 1.347, + "theoretical_loss": 3.4591594924401905, + "tokens_seen": 1811152896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589292929292929, + "loss": 1.4972, + "theoretical_loss": 3.459148851743578, + "tokens_seen": 1811218432 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045890909090909093, + "loss": 1.2867, + "theoretical_loss": 3.459138211539775, + "tokens_seen": 1811283968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588888888888889, + "loss": 1.4704, + "theoretical_loss": 3.4591275718287404, + "tokens_seen": 1811349504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588686868686869, + "loss": 1.27, + "theoretical_loss": 3.459116932610435, + "tokens_seen": 1811415040 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045884848484848486, + "loss": 1.3998, + "theoretical_loss": 3.4591062938848163, + "tokens_seen": 1811480576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588282828282828, + "loss": 1.4274, + "theoretical_loss": 3.4590956556518444, + "tokens_seen": 1811546112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588080808080808, + "loss": 1.5243, + "theoretical_loss": 3.4590850179114794, + "tokens_seen": 1811611648 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045878787878787883, + "loss": 1.3508, + "theoretical_loss": 3.45907438066368, + "tokens_seen": 1811677184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587676767676768, + "loss": 1.4852, + "theoretical_loss": 3.4590637439084055, + "tokens_seen": 1811742720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045874747474747475, + "loss": 1.3903, + "theoretical_loss": 3.459053107645615, + "tokens_seen": 1811808256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587272727272727, + "loss": 1.5327, + "theoretical_loss": 3.459042471875269, + "tokens_seen": 1811873792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587070707070707, + "loss": 1.202, + "theoretical_loss": 3.4590318365973256, + "tokens_seen": 1811939328 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045868686868686873, + "loss": 1.2374, + "theoretical_loss": 3.459021201811745, + "tokens_seen": 1812004864 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586666666666667, + "loss": 1.3289, + "theoretical_loss": 3.4590105675184866, + "tokens_seen": 1812070400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045864646464646464, + "loss": 1.4467, + "theoretical_loss": 3.458999933717509, + "tokens_seen": 1812135936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586262626262626, + "loss": 1.1531, + "theoretical_loss": 3.4589893004087733, + "tokens_seen": 1812201472 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045860606060606066, + "loss": 1.4899, + "theoretical_loss": 3.4589786675922367, + "tokens_seen": 1812267008 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.4100073277950287, + "objective/train/avg_token_score": 0.012066290713846684, + "objective/train/docs_used": 1024786, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3427298069000244, + "objective/train/original_loss": 4.287503242492676, + "objective/train/theoretical_loss": 3.4589680352678602, + "objective/train/tokens_used": 171191776, + "objective/train/unlikelihood_loss": 0.05709630250930786, + "theoretical_loss": 3.4589680352678602, + "tokens_seen": 1812332544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585858585858586, + "loss": 1.2515, + "theoretical_loss": 3.4589680352678602, + "tokens_seen": 1812332544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585656565656566, + "loss": 1.2303, + "theoretical_loss": 3.4589574034356025, + "tokens_seen": 1812398080 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045854545454545453, + "loss": 1.6304, + "theoretical_loss": 3.4589467720954232, + "tokens_seen": 1812463616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585252525252525, + "loss": 1.3892, + "theoretical_loss": 3.4589361412472828, + "tokens_seen": 1812529152 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045850505050505056, + "loss": 1.4854, + "theoretical_loss": 3.4589255108911385, + "tokens_seen": 1812594688 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584848484848485, + "loss": 1.5415, + "theoretical_loss": 3.458914881026952, + "tokens_seen": 1812660224 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045846464646464647, + "loss": 1.4956, + "theoretical_loss": 3.458904251654681, + "tokens_seen": 1812725760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584444444444444, + "loss": 1.3248, + "theoretical_loss": 3.458893622774286, + "tokens_seen": 1812791296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584242424242424, + "loss": 1.3537, + "theoretical_loss": 3.4588829943857258, + "tokens_seen": 1812856832 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045840404040404045, + "loss": 1.4755, + "theoretical_loss": 3.4588723664889605, + "tokens_seen": 1812922368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583838383838384, + "loss": 1.2218, + "theoretical_loss": 3.4588617390839493, + "tokens_seen": 1812987904 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045836363636363636, + "loss": 1.3251, + "theoretical_loss": 3.4588511121706516, + "tokens_seen": 1813053440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583434343434343, + "loss": 1.4598, + "theoretical_loss": 3.458840485749027, + "tokens_seen": 1813118976 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045832323232323233, + "loss": 1.6122, + "theoretical_loss": 3.4588298598190352, + "tokens_seen": 1813184512 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045830303030303034, + "loss": 1.1923, + "theoretical_loss": 3.458819234380635, + "tokens_seen": 1813250048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582828282828283, + "loss": 1.3857, + "theoretical_loss": 3.4588086094337864, + "tokens_seen": 1813315584 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045826262626262626, + "loss": 1.4029, + "theoretical_loss": 3.4587979849784487, + "tokens_seen": 1813381120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582424242424242, + "loss": 1.3769, + "theoretical_loss": 3.458787361014582, + "tokens_seen": 1813446656 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582222222222222, + "loss": 1.3538, + "theoretical_loss": 3.4587767375421445, + "tokens_seen": 1813512192 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045820202020202023, + "loss": 1.2557, + "theoretical_loss": 3.458766114561097, + "tokens_seen": 1813577728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581818181818182, + "loss": 1.4583, + "theoretical_loss": 3.458755492071398, + "tokens_seen": 1813643264 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045816161616161615, + "loss": 1.3099, + "theoretical_loss": 3.458744870073008, + "tokens_seen": 1813708800 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045814141414141416, + "loss": 1.2544, + "theoretical_loss": 3.458734248565886, + "tokens_seen": 1813774336 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581212121212121, + "loss": 1.461, + "theoretical_loss": 3.4587236275499915, + "tokens_seen": 1813839872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581010101010101, + "loss": 1.1234, + "theoretical_loss": 3.458713007025284, + "tokens_seen": 1813905408 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.4995112419128418, + "objective/train/avg_token_score": 0.0071546295657753944, + "objective/train/docs_used": 1026014, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7556525468826294, + "objective/train/original_loss": 3.839704990386963, + "objective/train/theoretical_loss": 3.4587023869917237, + "objective/train/tokens_used": 172830176, + "objective/train/unlikelihood_loss": 0.09379331767559052, + "theoretical_loss": 3.4587023869917237, + "tokens_seen": 1813970944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580808080808081, + "loss": 1.4608, + "theoretical_loss": 3.4587023869917237, + "tokens_seen": 1813970944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045806060606060604, + "loss": 1.3605, + "theoretical_loss": 3.458691767449269, + "tokens_seen": 1814036480 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045804040404040405, + "loss": 1.1711, + "theoretical_loss": 3.45868114839788, + "tokens_seen": 1814102016 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458020202020202, + "loss": 1.3351, + "theoretical_loss": 3.4586705298375167, + "tokens_seen": 1814167552 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458, + "loss": 1.5054, + "theoretical_loss": 3.4586599117681383, + "tokens_seen": 1814233088 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457979797979798, + "loss": 1.6297, + "theoretical_loss": 3.458649294189704, + "tokens_seen": 1814298624 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457959595959596, + "loss": 1.4436, + "theoretical_loss": 3.458638677102174, + "tokens_seen": 1814364160 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045793939393939394, + "loss": 1.3552, + "theoretical_loss": 3.4586280605055073, + "tokens_seen": 1814429696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579191919191919, + "loss": 1.5167, + "theoretical_loss": 3.458617444399664, + "tokens_seen": 1814495232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578989898989899, + "loss": 1.4367, + "theoretical_loss": 3.4586068287846032, + "tokens_seen": 1814560768 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045787878787878787, + "loss": 1.4934, + "theoretical_loss": 3.458596213660285, + "tokens_seen": 1814626304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578585858585859, + "loss": 1.3668, + "theoretical_loss": 3.4585855990266685, + "tokens_seen": 1814691840 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045783838383838384, + "loss": 1.2387, + "theoretical_loss": 3.4585749848837133, + "tokens_seen": 1814757376 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578181818181818, + "loss": 1.2531, + "theoretical_loss": 3.45856437123138, + "tokens_seen": 1814822912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577979797979798, + "loss": 1.4417, + "theoretical_loss": 3.4585537580696264, + "tokens_seen": 1814888448 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577777777777778, + "loss": 1.6412, + "theoretical_loss": 3.458543145398414, + "tokens_seen": 1814953984 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045775757575757577, + "loss": 1.6142, + "theoretical_loss": 3.458532533217701, + "tokens_seen": 1815019520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045773737373737373, + "loss": 1.5157, + "theoretical_loss": 3.4585219215274474, + "tokens_seen": 1815085056 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045771717171717174, + "loss": 1.43, + "theoretical_loss": 3.458511310327613, + "tokens_seen": 1815150592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576969696969697, + "loss": 1.2891, + "theoretical_loss": 3.458500699618158, + "tokens_seen": 1815216128 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576767676767677, + "loss": 1.4889, + "theoretical_loss": 3.458490089399041, + "tokens_seen": 1815281664 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045765656565656567, + "loss": 1.5082, + "theoretical_loss": 3.4584794796702223, + "tokens_seen": 1815347200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576363636363636, + "loss": 1.3993, + "theoretical_loss": 3.458468870431661, + "tokens_seen": 1815412736 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045761616161616163, + "loss": 1.4451, + "theoretical_loss": 3.4584582616833175, + "tokens_seen": 1815478272 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045759595959595964, + "loss": 1.3518, + "theoretical_loss": 3.458447653425151, + "tokens_seen": 1815543808 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.34109237790107727, + "objective/train/avg_token_score": 0.014077051542699337, + "objective/train/docs_used": 1026828, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3032065629959106, + "objective/train/original_loss": 4.5359601974487305, + "objective/train/theoretical_loss": 3.4584370456571207, + "objective/train/tokens_used": 174468576, + "objective/train/unlikelihood_loss": 0.057071033865213394, + "theoretical_loss": 3.4584370456571207, + "tokens_seen": 1815609344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575757575757576, + "loss": 1.4086, + "theoretical_loss": 3.4584370456571207, + "tokens_seen": 1815609344 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045755555555555556, + "loss": 1.3255, + "theoretical_loss": 3.4584264383791874, + "tokens_seen": 1815674880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575353535353535, + "loss": 1.5165, + "theoretical_loss": 3.4584158315913096, + "tokens_seen": 1815740416 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575151515151515, + "loss": 1.4522, + "theoretical_loss": 3.458405225293448, + "tokens_seen": 1815805952 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045749494949494954, + "loss": 1.4464, + "theoretical_loss": 3.458394619485561, + "tokens_seen": 1815871488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574747474747475, + "loss": 1.495, + "theoretical_loss": 3.4583840141676094, + "tokens_seen": 1815937024 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045745454545454545, + "loss": 1.6631, + "theoretical_loss": 3.458373409339553, + "tokens_seen": 1816002560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574343434343434, + "loss": 1.4861, + "theoretical_loss": 3.4583628050013506, + "tokens_seen": 1816068096 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574141414141415, + "loss": 1.312, + "theoretical_loss": 3.4583522011529624, + "tokens_seen": 1816133632 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045739393939393943, + "loss": 1.3567, + "theoretical_loss": 3.458341597794348, + "tokens_seen": 1816199168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004573737373737374, + "loss": 1.4321, + "theoretical_loss": 3.4583309949254675, + "tokens_seen": 1816264704 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045735353535353534, + "loss": 1.3804, + "theoretical_loss": 3.4583203925462795, + "tokens_seen": 1816330240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004573333333333333, + "loss": 1.292, + "theoretical_loss": 3.4583097906567453, + "tokens_seen": 1816395776 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045731313131313137, + "loss": 1.3965, + "theoretical_loss": 3.458299189256823, + "tokens_seen": 1816461312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572929292929293, + "loss": 1.532, + "theoretical_loss": 3.458288588346474, + "tokens_seen": 1816526848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572727272727273, + "loss": 1.5313, + "theoretical_loss": 3.4582779879256567, + "tokens_seen": 1816592384 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045725252525252524, + "loss": 1.314, + "theoretical_loss": 3.458267387994331, + "tokens_seen": 1816657920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045723232323232325, + "loss": 1.4601, + "theoretical_loss": 3.458256788552457, + "tokens_seen": 1816723456 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045721212121212126, + "loss": 1.169, + "theoretical_loss": 3.4582461895999947, + "tokens_seen": 1816788992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571919191919192, + "loss": 1.7361, + "theoretical_loss": 3.458235591136903, + "tokens_seen": 1816854528 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045717171717171717, + "loss": 1.4227, + "theoretical_loss": 3.458224993163143, + "tokens_seen": 1816920064 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045715151515151513, + "loss": 1.4055, + "theoretical_loss": 3.458214395678673, + "tokens_seen": 1816985600 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045713131313131314, + "loss": 1.37, + "theoretical_loss": 3.4582037986834537, + "tokens_seen": 1817051136 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045711111111111115, + "loss": 1.6846, + "theoretical_loss": 3.458193202177444, + "tokens_seen": 1817116672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004570909090909091, + "loss": 1.2868, + "theoretical_loss": 3.4581826061606047, + "tokens_seen": 1817182208 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.3228861093521118, + "objective/train/avg_token_score": 0.038064464926719666, + "objective/train/docs_used": 1028029, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0878517627716064, + "objective/train/original_loss": 4.760405540466309, + "objective/train/theoretical_loss": 3.458172010632895, + "objective/train/tokens_used": 176106976, + "objective/train/unlikelihood_loss": 0.04580136016011238, + "theoretical_loss": 3.458172010632895, + "tokens_seen": 1817247744 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045707070707070706, + "loss": 1.159, + "theoretical_loss": 3.458172010632895, + "tokens_seen": 1817247744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004570505050505051, + "loss": 1.4579, + "theoretical_loss": 3.458161415594275, + "tokens_seen": 1817313280 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045703030303030303, + "loss": 1.3157, + "theoretical_loss": 3.458150821044704, + "tokens_seen": 1817378816 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045701010101010104, + "loss": 1.324, + "theoretical_loss": 3.458140226984142, + "tokens_seen": 1817444352 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456989898989899, + "loss": 1.2537, + "theoretical_loss": 3.458129633412549, + "tokens_seen": 1817509888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045696969696969696, + "loss": 1.6337, + "theoretical_loss": 3.4581190403298847, + "tokens_seen": 1817575424 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045694949494949497, + "loss": 1.3909, + "theoretical_loss": 3.458108447736109, + "tokens_seen": 1817640960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569292929292929, + "loss": 1.3148, + "theoretical_loss": 3.4580978556311814, + "tokens_seen": 1817706496 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045690909090909094, + "loss": 1.2169, + "theoretical_loss": 3.458087264015062, + "tokens_seen": 1817772032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004568888888888889, + "loss": 1.5658, + "theoretical_loss": 3.45807667288771, + "tokens_seen": 1817837568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004568686868686869, + "loss": 1.5285, + "theoretical_loss": 3.4580660822490863, + "tokens_seen": 1817903104 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045684848484848486, + "loss": 1.2645, + "theoretical_loss": 3.45805549209915, + "tokens_seen": 1817968640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004568282828282828, + "loss": 1.5466, + "theoretical_loss": 3.4580449024378614, + "tokens_seen": 1818034176 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045680808080808083, + "loss": 1.4763, + "theoretical_loss": 3.4580343132651796, + "tokens_seen": 1818099712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567878787878788, + "loss": 1.3013, + "theoretical_loss": 3.458023724581065, + "tokens_seen": 1818165248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567676767676768, + "loss": 1.4691, + "theoretical_loss": 3.4580131363854782, + "tokens_seen": 1818230784 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045674747474747475, + "loss": 1.3829, + "theoretical_loss": 3.458002548678377, + "tokens_seen": 1818296320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567272727272727, + "loss": 1.4288, + "theoretical_loss": 3.457991961459723, + "tokens_seen": 1818361856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567070707070707, + "loss": 1.7387, + "theoretical_loss": 3.4579813747294756, + "tokens_seen": 1818427392 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045668686868686873, + "loss": 1.2347, + "theoretical_loss": 3.4579707884875948, + "tokens_seen": 1818492928 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566666666666667, + "loss": 1.3851, + "theoretical_loss": 3.4579602027340393, + "tokens_seen": 1818558464 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045664646464646465, + "loss": 1.4735, + "theoretical_loss": 3.457949617468771, + "tokens_seen": 1818624000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566262626262626, + "loss": 1.4051, + "theoretical_loss": 3.457939032691748, + "tokens_seen": 1818689536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566060606060606, + "loss": 1.4462, + "theoretical_loss": 3.4579284484029316, + "tokens_seen": 1818755072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565858585858586, + "loss": 1.2984, + "theoretical_loss": 3.45791786460228, + "tokens_seen": 1818820608 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.464809387922287, + "objective/train/avg_token_score": 0.016029013320803642, + "objective/train/docs_used": 1028792, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6073720455169678, + "objective/train/original_loss": 3.9546051025390625, + "objective/train/theoretical_loss": 3.4579072812897547, + "objective/train/tokens_used": 177745376, + "objective/train/unlikelihood_loss": 0.07868336141109467, + "theoretical_loss": 3.4579072812897547, + "tokens_seen": 1818886144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565656565656566, + "loss": 1.5138, + "theoretical_loss": 3.4579072812897547, + "tokens_seen": 1818886144 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045654545454545454, + "loss": 1.4139, + "theoretical_loss": 3.457896698465315, + "tokens_seen": 1818951680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565252525252525, + "loss": 1.5099, + "theoretical_loss": 3.457886116128921, + "tokens_seen": 1819017216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565050505050505, + "loss": 1.3184, + "theoretical_loss": 3.4578755342805323, + "tokens_seen": 1819082752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564848484848485, + "loss": 1.4564, + "theoretical_loss": 3.457864952920109, + "tokens_seen": 1819148288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564646464646465, + "loss": 1.4435, + "theoretical_loss": 3.4578543720476103, + "tokens_seen": 1819213824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045644444444444443, + "loss": 1.6601, + "theoretical_loss": 3.4578437916629974, + "tokens_seen": 1819279360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564242424242424, + "loss": 1.3769, + "theoretical_loss": 3.4578332117662294, + "tokens_seen": 1819344896 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045640404040404045, + "loss": 1.4961, + "theoretical_loss": 3.4578226323572663, + "tokens_seen": 1819410432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563838383838384, + "loss": 1.2399, + "theoretical_loss": 3.4578120534360686, + "tokens_seen": 1819475968 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045636363636363637, + "loss": 1.5018, + "theoretical_loss": 3.4578014750025954, + "tokens_seen": 1819541504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563434343434343, + "loss": 1.4262, + "theoretical_loss": 3.457790897056807, + "tokens_seen": 1819607040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563232323232323, + "loss": 1.4682, + "theoretical_loss": 3.4577803195986636, + "tokens_seen": 1819672576 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045630303030303035, + "loss": 1.5494, + "theoretical_loss": 3.457769742628125, + "tokens_seen": 1819738112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562828282828283, + "loss": 1.4778, + "theoretical_loss": 3.4577591661451517, + "tokens_seen": 1819803648 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045626262626262626, + "loss": 1.459, + "theoretical_loss": 3.457748590149703, + "tokens_seen": 1819869184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562424242424242, + "loss": 1.5946, + "theoretical_loss": 3.457738014641738, + "tokens_seen": 1819934720 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045622222222222223, + "loss": 1.3618, + "theoretical_loss": 3.4577274396212183, + "tokens_seen": 1820000256 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045620202020202024, + "loss": 1.2889, + "theoretical_loss": 3.457716865088103, + "tokens_seen": 1820065792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561818181818182, + "loss": 1.3204, + "theoretical_loss": 3.457706291042353, + "tokens_seen": 1820131328 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045616161616161615, + "loss": 1.4486, + "theoretical_loss": 3.4576957174839267, + "tokens_seen": 1820196864 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561414141414141, + "loss": 1.2579, + "theoretical_loss": 3.4576851444127854, + "tokens_seen": 1820262400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561212121212121, + "loss": 1.2631, + "theoretical_loss": 3.457674571828889, + "tokens_seen": 1820327936 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045610101010101013, + "loss": 1.2376, + "theoretical_loss": 3.457663999732197, + "tokens_seen": 1820393472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560808080808081, + "loss": 1.3863, + "theoretical_loss": 3.4576534281226694, + "tokens_seen": 1820459008 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.5623167157173157, + "objective/train/avg_token_score": 0.014118832536041737, + "objective/train/docs_used": 1029550, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7937812805175781, + "objective/train/original_loss": 3.694382429122925, + "objective/train/theoretical_loss": 3.457642857000267, + "objective/train/tokens_used": 179383776, + "objective/train/unlikelihood_loss": 0.07253962755203247, + "theoretical_loss": 3.457642857000267, + "tokens_seen": 1820524544 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045606060606060605, + "loss": 1.5616, + "theoretical_loss": 3.457642857000267, + "tokens_seen": 1820524544 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045604040404040406, + "loss": 1.4623, + "theoretical_loss": 3.457632286364949, + "tokens_seen": 1820590080 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045602020202020207, + "loss": 1.5794, + "theoretical_loss": 3.4576217162166754, + "tokens_seen": 1820655616 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456, + "loss": 1.4721, + "theoretical_loss": 3.457611146555407, + "tokens_seen": 1820721152 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455979797979798, + "loss": 1.5718, + "theoretical_loss": 3.457600577381103, + "tokens_seen": 1820786688 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045595959595959594, + "loss": 1.3938, + "theoretical_loss": 3.4575900086937237, + "tokens_seen": 1820852224 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045593939393939395, + "loss": 1.5038, + "theoretical_loss": 3.4575794404932294, + "tokens_seen": 1820917760 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045591919191919196, + "loss": 1.4119, + "theoretical_loss": 3.4575688727795804, + "tokens_seen": 1820983296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558989898989899, + "loss": 1.3345, + "theoretical_loss": 3.4575583055527357, + "tokens_seen": 1821048832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558787878787879, + "loss": 1.2874, + "theoretical_loss": 3.4575477388126563, + "tokens_seen": 1821114368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558585858585859, + "loss": 1.1633, + "theoretical_loss": 3.457537172559302, + "tokens_seen": 1821179904 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045583838383838384, + "loss": 1.4255, + "theoretical_loss": 3.4575266067926327, + "tokens_seen": 1821245440 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045581818181818185, + "loss": 1.4251, + "theoretical_loss": 3.4575160415126085, + "tokens_seen": 1821310976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557979797979798, + "loss": 1.4595, + "theoretical_loss": 3.4575054767191897, + "tokens_seen": 1821376512 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045577777777777777, + "loss": 1.4112, + "theoretical_loss": 3.4574949124123364, + "tokens_seen": 1821442048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557575757575758, + "loss": 1.3731, + "theoretical_loss": 3.457484348592008, + "tokens_seen": 1821507584 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045573737373737374, + "loss": 1.6005, + "theoretical_loss": 3.4574737852581654, + "tokens_seen": 1821573120 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045571717171717175, + "loss": 1.4437, + "theoretical_loss": 3.4574632224107686, + "tokens_seen": 1821638656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556969696969697, + "loss": 1.6006, + "theoretical_loss": 3.4574526600497775, + "tokens_seen": 1821704192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556767676767677, + "loss": 1.3622, + "theoretical_loss": 3.457442098175152, + "tokens_seen": 1821769728 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045565656565656567, + "loss": 1.4977, + "theoretical_loss": 3.457431536786853, + "tokens_seen": 1821835264 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045563636363636363, + "loss": 1.5852, + "theoretical_loss": 3.4574209758848395, + "tokens_seen": 1821900800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045561616161616164, + "loss": 1.3857, + "theoretical_loss": 3.457410415469072, + "tokens_seen": 1821966336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555959595959596, + "loss": 1.5078, + "theoretical_loss": 3.4573998555395113, + "tokens_seen": 1822031872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555757575757576, + "loss": 1.4179, + "theoretical_loss": 3.4573892960961166, + "tokens_seen": 1822097408 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.4111681282520294, + "objective/train/avg_token_score": 0.017115775495767593, + "objective/train/docs_used": 1030259, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2357131242752075, + "objective/train/original_loss": 3.788912296295166, + "objective/train/theoretical_loss": 3.457378737138849, + "objective/train/tokens_used": 181022176, + "objective/train/unlikelihood_loss": 0.0951184332370758, + "theoretical_loss": 3.457378737138849, + "tokens_seen": 1822162944 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045555555555555556, + "loss": 1.2408, + "theoretical_loss": 3.457378737138849, + "tokens_seen": 1822162944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555353535353535, + "loss": 1.3708, + "theoretical_loss": 3.4573681786676675, + "tokens_seen": 1822228480 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045551515151515153, + "loss": 1.5096, + "theoretical_loss": 3.457357620682533, + "tokens_seen": 1822294016 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045549494949494954, + "loss": 1.3569, + "theoretical_loss": 3.4573470631834056, + "tokens_seen": 1822359552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554747474747475, + "loss": 1.3497, + "theoretical_loss": 3.457336506170245, + "tokens_seen": 1822425088 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045545454545454546, + "loss": 1.5165, + "theoretical_loss": 3.457325949643012, + "tokens_seen": 1822490624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554343434343434, + "loss": 1.5752, + "theoretical_loss": 3.4573153936016663, + "tokens_seen": 1822556160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554141414141414, + "loss": 1.2614, + "theoretical_loss": 3.4573048380461686, + "tokens_seen": 1822621696 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045539393939393944, + "loss": 1.4106, + "theoretical_loss": 3.4572942829764783, + "tokens_seen": 1822687232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553737373737374, + "loss": 1.4145, + "theoretical_loss": 3.4572837283925564, + "tokens_seen": 1822752768 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045535353535353535, + "loss": 1.4457, + "theoretical_loss": 3.457273174294362, + "tokens_seen": 1822818304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553333333333333, + "loss": 1.418, + "theoretical_loss": 3.457262620681856, + "tokens_seen": 1822883840 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045531313131313137, + "loss": 1.3548, + "theoretical_loss": 3.4572520675549985, + "tokens_seen": 1822949376 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045529292929292933, + "loss": 1.4232, + "theoretical_loss": 3.45724151491375, + "tokens_seen": 1823014912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552727272727273, + "loss": 1.5722, + "theoretical_loss": 3.4572309627580706, + "tokens_seen": 1823080448 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045525252525252524, + "loss": 1.6458, + "theoretical_loss": 3.4572204110879197, + "tokens_seen": 1823145984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552323232323232, + "loss": 1.352, + "theoretical_loss": 3.4572098599032586, + "tokens_seen": 1823211520 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045521212121212126, + "loss": 1.6786, + "theoretical_loss": 3.4571993092040465, + "tokens_seen": 1823277056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551919191919192, + "loss": 1.6506, + "theoretical_loss": 3.4571887589902444, + "tokens_seen": 1823342592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551717171717172, + "loss": 1.2894, + "theoretical_loss": 3.457178209261812, + "tokens_seen": 1823408128 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045515151515151513, + "loss": 1.3039, + "theoretical_loss": 3.457167660018711, + "tokens_seen": 1823473664 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551313131313131, + "loss": 1.1383, + "theoretical_loss": 3.4571571112608988, + "tokens_seen": 1823539200 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045511111111111116, + "loss": 1.3979, + "theoretical_loss": 3.4571465629883384, + "tokens_seen": 1823604736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550909090909091, + "loss": 1.3313, + "theoretical_loss": 3.4571360152009882, + "tokens_seen": 1823670272 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045507070707070707, + "loss": 1.5223, + "theoretical_loss": 3.4571254678988095, + "tokens_seen": 1823735808 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.38483625650405884, + "objective/train/avg_token_score": 0.019340477883815765, + "objective/train/docs_used": 1031478, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2517280578613281, + "objective/train/original_loss": 4.015500068664551, + "objective/train/theoretical_loss": 3.4571149210817618, + "objective/train/tokens_used": 182660576, + "objective/train/unlikelihood_loss": 0.09212837368249893, + "theoretical_loss": 3.4571149210817618, + "tokens_seen": 1823801344 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045505050505050503, + "loss": 1.3643, + "theoretical_loss": 3.4571149210817618, + "tokens_seen": 1823801344 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045503030303030304, + "loss": 1.4704, + "theoretical_loss": 3.4571043747498065, + "tokens_seen": 1823866880 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045501010101010105, + "loss": 1.5384, + "theoretical_loss": 3.4570938289029023, + "tokens_seen": 1823932416 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454989898989899, + "loss": 1.518, + "theoretical_loss": 3.457083283541011, + "tokens_seen": 1823997952 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045496969696969696, + "loss": 1.1422, + "theoretical_loss": 3.4570727386640914, + "tokens_seen": 1824063488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549494949494949, + "loss": 1.3168, + "theoretical_loss": 3.4570621942721047, + "tokens_seen": 1824129024 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045492929292929293, + "loss": 1.3456, + "theoretical_loss": 3.4570516503650115, + "tokens_seen": 1824194560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045490909090909094, + "loss": 1.4976, + "theoretical_loss": 3.4570411069427713, + "tokens_seen": 1824260096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548888888888889, + "loss": 1.4185, + "theoretical_loss": 3.4570305640053443, + "tokens_seen": 1824325632 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045486868686868686, + "loss": 1.3554, + "theoretical_loss": 3.457020021552691, + "tokens_seen": 1824391168 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045484848484848487, + "loss": 1.4911, + "theoretical_loss": 3.4570094795847726, + "tokens_seen": 1824456704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548282828282828, + "loss": 1.3135, + "theoretical_loss": 3.456998938101548, + "tokens_seen": 1824522240 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045480808080808083, + "loss": 1.0882, + "theoretical_loss": 3.456988397102979, + "tokens_seen": 1824587776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547878787878788, + "loss": 1.3685, + "theoretical_loss": 3.4569778565890243, + "tokens_seen": 1824653312 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045476767676767675, + "loss": 1.4268, + "theoretical_loss": 3.456967316559645, + "tokens_seen": 1824718848 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045474747474747476, + "loss": 1.5668, + "theoretical_loss": 3.4569567770148018, + "tokens_seen": 1824784384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547272727272727, + "loss": 1.6636, + "theoretical_loss": 3.4569462379544547, + "tokens_seen": 1824849920 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045470707070707073, + "loss": 1.4786, + "theoretical_loss": 3.4569356993785636, + "tokens_seen": 1824915456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546868686868687, + "loss": 1.3914, + "theoretical_loss": 3.456925161287089, + "tokens_seen": 1824980992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546666666666667, + "loss": 1.2798, + "theoretical_loss": 3.456914623679992, + "tokens_seen": 1825046528 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045464646464646465, + "loss": 1.4608, + "theoretical_loss": 3.4569040865572322, + "tokens_seen": 1825112064 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546262626262626, + "loss": 1.6782, + "theoretical_loss": 3.4568935499187696, + "tokens_seen": 1825177600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546060606060606, + "loss": 1.4233, + "theoretical_loss": 3.456883013764566, + "tokens_seen": 1825243136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545858585858586, + "loss": 1.5035, + "theoretical_loss": 3.4568724780945796, + "tokens_seen": 1825308672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545656565656566, + "loss": 1.4682, + "theoretical_loss": 3.456861942908773, + "tokens_seen": 1825374208 + }, + { + "debugging/Self-BLEU-5": 0.383412367211192, + "debugging/distinct-1-grams": 0.7526443544193546, + "debugging/distinct-2-grams": 0.9328595808568966, + "debugging/entropy-1-grams": 5.657827037788948, + "debugging/entropy-2-grams": 6.397750202501998, + "debugging/length": 484.77777777777777, + "debugging/num_segments": 9, + "epoch": 0.1, + "objective/train/aligned_token_count": 0.5272482633590698, + "objective/train/avg_token_score": 0.014473862946033478, + "objective/train/docs_used": 1032081, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6364436149597168, + "objective/train/original_loss": 3.784334897994995, + "objective/train/theoretical_loss": 3.456851408207105, + "objective/train/tokens_used": 184298976, + "objective/train/unlikelihood_loss": 0.08270136266946793, + "theoretical_loss": 3.456851408207105, + "tokens_seen": 1825439744 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045454545454545455, + "loss": 1.618, + "theoretical_loss": 3.456851408207105, + "tokens_seen": 1825439744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545252525252525, + "loss": 1.2274, + "theoretical_loss": 3.456840873989537, + "tokens_seen": 1825505280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545050505050505, + "loss": 1.3231, + "theoretical_loss": 3.4568303402560288, + "tokens_seen": 1825570816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544848484848485, + "loss": 1.3608, + "theoretical_loss": 3.4568198070065406, + "tokens_seen": 1825636352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544646464646465, + "loss": 1.4591, + "theoretical_loss": 3.4568092742410332, + "tokens_seen": 1825701888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045444444444444444, + "loss": 1.2171, + "theoretical_loss": 3.456798741959467, + "tokens_seen": 1825767424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544242424242424, + "loss": 1.4365, + "theoretical_loss": 3.456788210161802, + "tokens_seen": 1825832960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544040404040404, + "loss": 1.4114, + "theoretical_loss": 3.456777678847999, + "tokens_seen": 1825898496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543838383838384, + "loss": 1.3937, + "theoretical_loss": 3.4567671480180184, + "tokens_seen": 1825964032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543636363636364, + "loss": 1.4272, + "theoretical_loss": 3.4567566176718203, + "tokens_seen": 1826029568 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045434343434343433, + "loss": 1.5479, + "theoretical_loss": 3.456746087809365, + "tokens_seen": 1826095104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543232323232323, + "loss": 1.1661, + "theoretical_loss": 3.456735558430614, + "tokens_seen": 1826160640 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045430303030303035, + "loss": 1.1125, + "theoretical_loss": 3.4567250295355265, + "tokens_seen": 1826226176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542828282828283, + "loss": 1.4769, + "theoretical_loss": 3.456714501124063, + "tokens_seen": 1826291712 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045426262626262627, + "loss": 1.3936, + "theoretical_loss": 3.4567039731961846, + "tokens_seen": 1826357248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542424242424242, + "loss": 1.2966, + "theoretical_loss": 3.4566934457518514, + "tokens_seen": 1826422784 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045422222222222223, + "loss": 1.341, + "theoretical_loss": 3.456682918791024, + "tokens_seen": 1826488320 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045420202020202025, + "loss": 1.4049, + "theoretical_loss": 3.4566723923136626, + "tokens_seen": 1826553856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541818181818182, + "loss": 1.4241, + "theoretical_loss": 3.4566618663197275, + "tokens_seen": 1826619392 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045416161616161616, + "loss": 1.3875, + "theoretical_loss": 3.45665134080918, + "tokens_seen": 1826684928 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541414141414141, + "loss": 1.5876, + "theoretical_loss": 3.4566408157819795, + "tokens_seen": 1826750464 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541212121212122, + "loss": 1.3572, + "theoretical_loss": 3.456630291238087, + "tokens_seen": 1826816000 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045410101010101014, + "loss": 1.5581, + "theoretical_loss": 3.4566197671774628, + "tokens_seen": 1826881536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540808080808081, + "loss": 1.4737, + "theoretical_loss": 3.456609243600068, + "tokens_seen": 1826947072 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045406060606060605, + "loss": 1.428, + "theoretical_loss": 3.456598720505862, + "tokens_seen": 1827012608 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.3259408473968506, + "objective/train/avg_token_score": 0.02392873726785183, + "objective/train/docs_used": 1033277, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0870252847671509, + "objective/train/original_loss": 3.9227612018585205, + "objective/train/theoretical_loss": 3.4565881978948063, + "objective/train/tokens_used": 185937376, + "objective/train/unlikelihood_loss": 0.08083102852106094, + "theoretical_loss": 3.4565881978948063, + "tokens_seen": 1827078144 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454040404040404, + "loss": 1.1697, + "theoretical_loss": 3.4565881978948063, + "tokens_seen": 1827078144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540202020202021, + "loss": 1.3098, + "theoretical_loss": 3.4565776757668605, + "tokens_seen": 1827143680 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045400000000000003, + "loss": 1.3512, + "theoretical_loss": 3.456567154121986, + "tokens_seen": 1827209216 + }, + { + "epoch": 0.1, + "learning_rate": 0.000453979797979798, + "loss": 1.438, + "theoretical_loss": 3.4565566329601425, + "tokens_seen": 1827274752 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045395959595959594, + "loss": 1.6695, + "theoretical_loss": 3.4565461122812904, + "tokens_seen": 1827340288 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045393939393939396, + "loss": 1.1057, + "theoretical_loss": 3.4565355920853915, + "tokens_seen": 1827405824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045391919191919197, + "loss": 1.5515, + "theoretical_loss": 3.456525072372405, + "tokens_seen": 1827471360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538989898989899, + "loss": 1.2735, + "theoretical_loss": 3.456514553142292, + "tokens_seen": 1827536896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538787878787879, + "loss": 1.3733, + "theoretical_loss": 3.456504034395013, + "tokens_seen": 1827602432 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045385858585858584, + "loss": 1.3649, + "theoretical_loss": 3.4564935161305277, + "tokens_seen": 1827667968 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045383838383838385, + "loss": 1.7118, + "theoretical_loss": 3.4564829983487977, + "tokens_seen": 1827733504 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045381818181818186, + "loss": 1.3295, + "theoretical_loss": 3.4564724810497838, + "tokens_seen": 1827799040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537979797979798, + "loss": 1.2664, + "theoretical_loss": 3.456461964233445, + "tokens_seen": 1827864576 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537777777777778, + "loss": 1.5361, + "theoretical_loss": 3.4564514478997435, + "tokens_seen": 1827930112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537575757575758, + "loss": 1.5431, + "theoretical_loss": 3.456440932048639, + "tokens_seen": 1827995648 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045373737373737374, + "loss": 1.377, + "theoretical_loss": 3.4564304166800914, + "tokens_seen": 1828061184 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045371717171717175, + "loss": 1.3454, + "theoretical_loss": 3.456419901794063, + "tokens_seen": 1828126720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536969696969697, + "loss": 1.4446, + "theoretical_loss": 3.4564093873905124, + "tokens_seen": 1828192256 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045367676767676767, + "loss": 1.4514, + "theoretical_loss": 3.4563988734694018, + "tokens_seen": 1828257792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536565656565657, + "loss": 1.3498, + "theoretical_loss": 3.456388360030691, + "tokens_seen": 1828323328 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045363636363636363, + "loss": 1.4994, + "theoretical_loss": 3.4563778470743403, + "tokens_seen": 1828388864 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045361616161616164, + "loss": 1.1171, + "theoretical_loss": 3.456367334600311, + "tokens_seen": 1828454400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535959595959596, + "loss": 1.37, + "theoretical_loss": 3.4563568226085635, + "tokens_seen": 1828519936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535757575757576, + "loss": 1.4101, + "theoretical_loss": 3.456346311099058, + "tokens_seen": 1828585472 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045355555555555557, + "loss": 1.2472, + "theoretical_loss": 3.4563358000717552, + "tokens_seen": 1828651008 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.3474462330341339, + "objective/train/avg_token_score": 0.01256563700735569, + "objective/train/docs_used": 1034021, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1423338651657104, + "objective/train/original_loss": 4.592188835144043, + "objective/train/theoretical_loss": 3.456325289526616, + "objective/train/tokens_used": 187575776, + "objective/train/unlikelihood_loss": 0.06064225360751152, + "theoretical_loss": 3.456325289526616, + "tokens_seen": 1828716544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535353535353535, + "loss": 1.4799, + "theoretical_loss": 3.456325289526616, + "tokens_seen": 1828716544 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045351515151515154, + "loss": 1.2678, + "theoretical_loss": 3.456314779463601, + "tokens_seen": 1828782080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534949494949495, + "loss": 1.3876, + "theoretical_loss": 3.4563042698826707, + "tokens_seen": 1828847616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534747474747475, + "loss": 1.4213, + "theoretical_loss": 3.456293760783786, + "tokens_seen": 1828913152 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045345454545454546, + "loss": 1.596, + "theoretical_loss": 3.456283252166906, + "tokens_seen": 1828978688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534343434343434, + "loss": 1.253, + "theoretical_loss": 3.4562727440319936, + "tokens_seen": 1829044224 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045341414141414143, + "loss": 1.4315, + "theoretical_loss": 3.4562622363790076, + "tokens_seen": 1829109760 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045339393939393944, + "loss": 1.1965, + "theoretical_loss": 3.45625172920791, + "tokens_seen": 1829175296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533737373737374, + "loss": 1.3171, + "theoretical_loss": 3.45624122251866, + "tokens_seen": 1829240832 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045335353535353535, + "loss": 1.2845, + "theoretical_loss": 3.45623071631122, + "tokens_seen": 1829306368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533333333333333, + "loss": 1.6198, + "theoretical_loss": 3.456220210585549, + "tokens_seen": 1829371904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533131313131313, + "loss": 1.268, + "theoretical_loss": 3.4562097053416085, + "tokens_seen": 1829437440 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045329292929292933, + "loss": 1.3782, + "theoretical_loss": 3.456199200579359, + "tokens_seen": 1829502976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532727272727273, + "loss": 1.4489, + "theoretical_loss": 3.456188696298761, + "tokens_seen": 1829568512 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045325252525252525, + "loss": 1.3057, + "theoretical_loss": 3.4561781924997756, + "tokens_seen": 1829634048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532323232323232, + "loss": 1.5815, + "theoretical_loss": 3.456167689182363, + "tokens_seen": 1829699584 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532121212121212, + "loss": 1.5887, + "theoretical_loss": 3.456157186346484, + "tokens_seen": 1829765120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531919191919192, + "loss": 1.5131, + "theoretical_loss": 3.4561466839921, + "tokens_seen": 1829830656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531717171717172, + "loss": 1.6467, + "theoretical_loss": 3.4561361821191703, + "tokens_seen": 1829896192 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045315151515151514, + "loss": 1.54, + "theoretical_loss": 3.456125680727656, + "tokens_seen": 1829961728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531313131313131, + "loss": 1.6346, + "theoretical_loss": 3.4561151798175187, + "tokens_seen": 1830027264 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045311111111111116, + "loss": 1.2857, + "theoretical_loss": 3.4561046793887185, + "tokens_seen": 1830092800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530909090909091, + "loss": 1.1951, + "theoretical_loss": 3.456094179441216, + "tokens_seen": 1830158336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530707070707071, + "loss": 1.2465, + "theoretical_loss": 3.456083679974972, + "tokens_seen": 1830223872 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045305050505050503, + "loss": 1.3245, + "theoretical_loss": 3.456073180989947, + "tokens_seen": 1830289408 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.4240591526031494, + "objective/train/avg_token_score": 0.011298497207462788, + "objective/train/docs_used": 1035168, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4181580543518066, + "objective/train/original_loss": 4.181857585906982, + "objective/train/theoretical_loss": 3.456062682486102, + "objective/train/tokens_used": 189214176, + "objective/train/unlikelihood_loss": 0.05961797386407852, + "theoretical_loss": 3.456062682486102, + "tokens_seen": 1830354944 + }, + { + "epoch": 0.1, + "learning_rate": 0.000453030303030303, + "loss": 1.5355, + "theoretical_loss": 3.456062682486102, + "tokens_seen": 1830354944 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045301010101010106, + "loss": 1.466, + "theoretical_loss": 3.4560521844633976, + "tokens_seen": 1830420480 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452989898989899, + "loss": 1.5066, + "theoretical_loss": 3.456041686921795, + "tokens_seen": 1830486016 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045296969696969697, + "loss": 1.3351, + "theoretical_loss": 3.456031189861254, + "tokens_seen": 1830551552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529494949494949, + "loss": 1.2997, + "theoretical_loss": 3.4560206932817357, + "tokens_seen": 1830617088 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045292929292929294, + "loss": 1.5116, + "theoretical_loss": 3.4560101971832013, + "tokens_seen": 1830682624 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045290909090909095, + "loss": 1.0942, + "theoretical_loss": 3.455999701565611, + "tokens_seen": 1830748160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528888888888889, + "loss": 1.3721, + "theoretical_loss": 3.455989206428926, + "tokens_seen": 1830813696 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045286868686868686, + "loss": 1.2505, + "theoretical_loss": 3.455978711773107, + "tokens_seen": 1830879232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528484848484848, + "loss": 1.6109, + "theoretical_loss": 3.455968217598114, + "tokens_seen": 1830944768 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045282828282828283, + "loss": 1.3838, + "theoretical_loss": 3.455957723903909, + "tokens_seen": 1831010304 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045280808080808084, + "loss": 1.2037, + "theoretical_loss": 3.4559472306904517, + "tokens_seen": 1831075840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004527878787878788, + "loss": 1.3973, + "theoretical_loss": 3.4559367379577033, + "tokens_seen": 1831141376 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045276767676767675, + "loss": 1.4359, + "theoretical_loss": 3.4559262457056246, + "tokens_seen": 1831206912 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045274747474747477, + "loss": 1.6251, + "theoretical_loss": 3.4559157539341765, + "tokens_seen": 1831272448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004527272727272727, + "loss": 1.4879, + "theoretical_loss": 3.455905262643319, + "tokens_seen": 1831337984 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045270707070707073, + "loss": 1.7081, + "theoretical_loss": 3.4558947718330137, + "tokens_seen": 1831403520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526868686868687, + "loss": 1.4078, + "theoretical_loss": 3.4558842815032214, + "tokens_seen": 1831469056 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045266666666666665, + "loss": 1.5619, + "theoretical_loss": 3.455873791653903, + "tokens_seen": 1831534592 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045264646464646466, + "loss": 1.4385, + "theoretical_loss": 3.4558633022850183, + "tokens_seen": 1831600128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526262626262626, + "loss": 1.1827, + "theoretical_loss": 3.455852813396529, + "tokens_seen": 1831665664 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526060606060606, + "loss": 1.5327, + "theoretical_loss": 3.455842324988396, + "tokens_seen": 1831731200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525858585858586, + "loss": 1.6505, + "theoretical_loss": 3.455831837060579, + "tokens_seen": 1831796736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525656565656566, + "loss": 1.2537, + "theoretical_loss": 3.45582134961304, + "tokens_seen": 1831862272 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045254545454545455, + "loss": 1.2179, + "theoretical_loss": 3.4558108626457402, + "tokens_seen": 1831927808 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.2812194526195526, + "objective/train/avg_token_score": 0.04614793136715889, + "objective/train/docs_used": 1035908, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9016737937927246, + "objective/train/original_loss": 4.561851978302002, + "objective/train/theoretical_loss": 3.455800376158639, + "objective/train/tokens_used": 190852576, + "objective/train/unlikelihood_loss": 0.06523799896240234, + "theoretical_loss": 3.455800376158639, + "tokens_seen": 1831993344 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045252525252525256, + "loss": 1.376, + "theoretical_loss": 3.455800376158639, + "tokens_seen": 1831993344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525050505050505, + "loss": 1.4152, + "theoretical_loss": 3.4557898901516984, + "tokens_seen": 1832058880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524848484848485, + "loss": 1.565, + "theoretical_loss": 3.455779404624878, + "tokens_seen": 1832124416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524646464646465, + "loss": 1.4454, + "theoretical_loss": 3.45576891957814, + "tokens_seen": 1832189952 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045244444444444444, + "loss": 1.6646, + "theoretical_loss": 3.4557584350114445, + "tokens_seen": 1832255488 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045242424242424245, + "loss": 1.2104, + "theoretical_loss": 3.4557479509247524, + "tokens_seen": 1832321024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524040404040404, + "loss": 1.6734, + "theoretical_loss": 3.455737467318025, + "tokens_seen": 1832386560 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523838383838384, + "loss": 1.4283, + "theoretical_loss": 3.4557269841912226, + "tokens_seen": 1832452096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523636363636364, + "loss": 1.5698, + "theoretical_loss": 3.4557165015443063, + "tokens_seen": 1832517632 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045234343434343434, + "loss": 1.1731, + "theoretical_loss": 3.455706019377237, + "tokens_seen": 1832583168 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045232323232323235, + "loss": 1.2459, + "theoretical_loss": 3.4556955376899756, + "tokens_seen": 1832648704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523030303030303, + "loss": 1.3951, + "theoretical_loss": 3.4556850564824826, + "tokens_seen": 1832714240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004522828282828283, + "loss": 1.247, + "theoretical_loss": 3.4556745757547196, + "tokens_seen": 1832779776 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045226262626262627, + "loss": 1.4864, + "theoretical_loss": 3.455664095506647, + "tokens_seen": 1832845312 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045224242424242423, + "loss": 1.4557, + "theoretical_loss": 3.4556536157382256, + "tokens_seen": 1832910848 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045222222222222224, + "loss": 1.4369, + "theoretical_loss": 3.455643136449417, + "tokens_seen": 1832976384 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045220202020202025, + "loss": 1.4263, + "theoretical_loss": 3.455632657640181, + "tokens_seen": 1833041920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521818181818182, + "loss": 1.458, + "theoretical_loss": 3.4556221793104793, + "tokens_seen": 1833107456 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045216161616161616, + "loss": 1.6213, + "theoretical_loss": 3.4556117014602727, + "tokens_seen": 1833172992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521414141414141, + "loss": 1.3254, + "theoretical_loss": 3.455601224089522, + "tokens_seen": 1833238528 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045212121212121213, + "loss": 1.5437, + "theoretical_loss": 3.4555907471981886, + "tokens_seen": 1833304064 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045210101010101014, + "loss": 1.3216, + "theoretical_loss": 3.455580270786232, + "tokens_seen": 1833369600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520808080808081, + "loss": 1.459, + "theoretical_loss": 3.455569794853615, + "tokens_seen": 1833435136 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045206060606060606, + "loss": 1.4695, + "theoretical_loss": 3.4555593194002974, + "tokens_seen": 1833500672 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452040404040404, + "loss": 1.392, + "theoretical_loss": 3.4555488444262403, + "tokens_seen": 1833566208 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.48057183623313904, + "objective/train/avg_token_score": 0.015353728085756302, + "objective/train/docs_used": 1036404, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4945865869522095, + "objective/train/original_loss": 4.170429706573486, + "objective/train/theoretical_loss": 3.4555383699314044, + "objective/train/tokens_used": 192490976, + "objective/train/unlikelihood_loss": 0.0738421231508255, + "theoretical_loss": 3.4555383699314044, + "tokens_seen": 1833631744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520202020202021, + "loss": 1.5297, + "theoretical_loss": 3.4555383699314044, + "tokens_seen": 1833631744 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045200000000000004, + "loss": 1.5953, + "theoretical_loss": 3.4555278959157514, + "tokens_seen": 1833697280 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451979797979798, + "loss": 1.4807, + "theoretical_loss": 3.455517422379242, + "tokens_seen": 1833762816 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045195959595959595, + "loss": 1.4942, + "theoretical_loss": 3.455506949321837, + "tokens_seen": 1833828352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519393939393939, + "loss": 1.3145, + "theoretical_loss": 3.455496476743497, + "tokens_seen": 1833893888 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045191919191919197, + "loss": 1.5973, + "theoretical_loss": 3.4554860046441833, + "tokens_seen": 1833959424 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045189898989898993, + "loss": 1.3642, + "theoretical_loss": 3.455475533023857, + "tokens_seen": 1834024960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518787878787879, + "loss": 1.2289, + "theoretical_loss": 3.455465061882479, + "tokens_seen": 1834090496 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045185858585858584, + "loss": 1.5492, + "theoretical_loss": 3.4554545912200103, + "tokens_seen": 1834156032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518383838383838, + "loss": 1.2268, + "theoretical_loss": 3.455444121036412, + "tokens_seen": 1834221568 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045181818181818186, + "loss": 1.3147, + "theoretical_loss": 3.455433651331645, + "tokens_seen": 1834287104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004517979797979798, + "loss": 1.4772, + "theoretical_loss": 3.45542318210567, + "tokens_seen": 1834352640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004517777777777778, + "loss": 1.4504, + "theoretical_loss": 3.455412713358448, + "tokens_seen": 1834418176 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045175757575757574, + "loss": 1.5777, + "theoretical_loss": 3.45540224508994, + "tokens_seen": 1834483712 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045173737373737375, + "loss": 1.4718, + "theoretical_loss": 3.455391777300108, + "tokens_seen": 1834549248 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045171717171717176, + "loss": 1.5121, + "theoretical_loss": 3.455381309988912, + "tokens_seen": 1834614784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516969696969697, + "loss": 1.544, + "theoretical_loss": 3.455370843156313, + "tokens_seen": 1834680320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045167676767676767, + "loss": 1.6024, + "theoretical_loss": 3.4553603768022727, + "tokens_seen": 1834745856 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045165656565656563, + "loss": 1.4055, + "theoretical_loss": 3.4553499109267514, + "tokens_seen": 1834811392 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045163636363636364, + "loss": 1.3813, + "theoretical_loss": 3.4553394455297104, + "tokens_seen": 1834876928 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045161616161616165, + "loss": 1.4046, + "theoretical_loss": 3.455328980611111, + "tokens_seen": 1834942464 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515959595959596, + "loss": 1.5633, + "theoretical_loss": 3.455318516170914, + "tokens_seen": 1835008000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045157575757575756, + "loss": 1.2908, + "theoretical_loss": 3.4553080522090807, + "tokens_seen": 1835073536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515555555555556, + "loss": 1.2658, + "theoretical_loss": 3.4552975887255717, + "tokens_seen": 1835139072 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045153535353535353, + "loss": 1.0296, + "theoretical_loss": 3.455287125720348, + "tokens_seen": 1835204608 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.46584799885749817, + "objective/train/avg_token_score": 0.026970431208610535, + "objective/train/docs_used": 1037817, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6634101867675781, + "objective/train/original_loss": 4.356515884399414, + "objective/train/theoretical_loss": 3.455276663193372, + "objective/train/tokens_used": 194129376, + "objective/train/unlikelihood_loss": 0.06059441342949867, + "theoretical_loss": 3.455276663193372, + "tokens_seen": 1835270144 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045151515151515154, + "loss": 1.435, + "theoretical_loss": 3.455276663193372, + "tokens_seen": 1835270144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004514949494949495, + "loss": 1.6169, + "theoretical_loss": 3.4552662011446023, + "tokens_seen": 1835335680 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045147474747474746, + "loss": 1.4235, + "theoretical_loss": 3.4552557395740022, + "tokens_seen": 1835401216 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045145454545454547, + "loss": 1.4122, + "theoretical_loss": 3.4552452784815317, + "tokens_seen": 1835466752 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004514343434343434, + "loss": 1.5246, + "theoretical_loss": 3.455234817867152, + "tokens_seen": 1835532288 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045141414141414144, + "loss": 1.4689, + "theoretical_loss": 3.455224357730825, + "tokens_seen": 1835597824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513939393939394, + "loss": 1.2776, + "theoretical_loss": 3.4552138980725102, + "tokens_seen": 1835663360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513737373737374, + "loss": 1.3385, + "theoretical_loss": 3.4552034388921697, + "tokens_seen": 1835728896 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045135353535353536, + "loss": 1.3866, + "theoretical_loss": 3.4551929801897647, + "tokens_seen": 1835794432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513333333333333, + "loss": 1.5819, + "theoretical_loss": 3.455182521965256, + "tokens_seen": 1835859968 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045131313131313133, + "loss": 1.3853, + "theoretical_loss": 3.455172064218605, + "tokens_seen": 1835925504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512929292929293, + "loss": 1.3696, + "theoretical_loss": 3.4551616069497726, + "tokens_seen": 1835991040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512727272727273, + "loss": 1.2499, + "theoretical_loss": 3.4551511501587195, + "tokens_seen": 1836056576 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045125252525252525, + "loss": 1.5628, + "theoretical_loss": 3.4551406938454075, + "tokens_seen": 1836122112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512323232323232, + "loss": 1.6587, + "theoretical_loss": 3.4551302380097972, + "tokens_seen": 1836187648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512121212121212, + "loss": 1.4389, + "theoretical_loss": 3.4551197826518507, + "tokens_seen": 1836253184 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045119191919191923, + "loss": 1.3724, + "theoretical_loss": 3.4551093277715275, + "tokens_seen": 1836318720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511717171717172, + "loss": 1.3688, + "theoretical_loss": 3.45509887336879, + "tokens_seen": 1836384256 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045115151515151515, + "loss": 1.4629, + "theoretical_loss": 3.455088419443599, + "tokens_seen": 1836449792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511313131313131, + "loss": 1.4199, + "theoretical_loss": 3.4550779659959154, + "tokens_seen": 1836515328 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511111111111111, + "loss": 1.4308, + "theoretical_loss": 3.455067513025701, + "tokens_seen": 1836580864 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510909090909091, + "loss": 1.4939, + "theoretical_loss": 3.4550570605329165, + "tokens_seen": 1836646400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510707070707071, + "loss": 1.5078, + "theoretical_loss": 3.455046608517523, + "tokens_seen": 1836711936 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045105050505050504, + "loss": 1.5747, + "theoretical_loss": 3.4550361569794816, + "tokens_seen": 1836777472 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451030303030303, + "loss": 1.4927, + "theoretical_loss": 3.4550257059187537, + "tokens_seen": 1836843008 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.5, + "objective/train/avg_token_score": 0.013788322918117046, + "objective/train/docs_used": 1038484, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6846576929092407, + "objective/train/original_loss": 3.9736297130584717, + "objective/train/theoretical_loss": 3.4550152553353, + "objective/train/tokens_used": 195767776, + "objective/train/unlikelihood_loss": 0.05097836256027222, + "theoretical_loss": 3.4550152553353, + "tokens_seen": 1836908544 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045101010101010106, + "loss": 1.3611, + "theoretical_loss": 3.4550152553353, + "tokens_seen": 1836908544 + }, + { + "epoch": 0.11, + "learning_rate": 0.000450989898989899, + "loss": 1.3674, + "theoretical_loss": 3.4550048052290823, + "tokens_seen": 1836974080 + }, + { + "epoch": 0.11, + "learning_rate": 0.000450969696969697, + "loss": 1.3659, + "theoretical_loss": 3.454994355600062, + "tokens_seen": 1837039616 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045094949494949493, + "loss": 1.2901, + "theoretical_loss": 3.4549839064481995, + "tokens_seen": 1837105152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509292929292929, + "loss": 1.4305, + "theoretical_loss": 3.4549734577734563, + "tokens_seen": 1837170688 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045090909090909095, + "loss": 1.45, + "theoretical_loss": 3.4549630095757937, + "tokens_seen": 1837236224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508888888888889, + "loss": 1.379, + "theoretical_loss": 3.4549525618551726, + "tokens_seen": 1837301760 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045086868686868687, + "loss": 1.6066, + "theoretical_loss": 3.454942114611555, + "tokens_seen": 1837367296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508484848484848, + "loss": 1.4182, + "theoretical_loss": 3.454931667844901, + "tokens_seen": 1837432832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508282828282829, + "loss": 1.3809, + "theoretical_loss": 3.4549212215551726, + "tokens_seen": 1837498368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045080808080808085, + "loss": 1.4271, + "theoretical_loss": 3.4549107757423307, + "tokens_seen": 1837563904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507878787878788, + "loss": 1.4629, + "theoretical_loss": 3.4549003304063364, + "tokens_seen": 1837629440 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045076767676767676, + "loss": 1.6347, + "theoretical_loss": 3.4548898855471517, + "tokens_seen": 1837694976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507474747474747, + "loss": 1.4147, + "theoretical_loss": 3.4548794411647368, + "tokens_seen": 1837760512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507272727272728, + "loss": 1.561, + "theoretical_loss": 3.4548689972590534, + "tokens_seen": 1837826048 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045070707070707074, + "loss": 1.7009, + "theoretical_loss": 3.4548585538300625, + "tokens_seen": 1837891584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506868686868687, + "loss": 1.5077, + "theoretical_loss": 3.454848110877726, + "tokens_seen": 1837957120 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045066666666666665, + "loss": 1.3909, + "theoretical_loss": 3.4548376684020043, + "tokens_seen": 1838022656 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045064646464646466, + "loss": 1.4584, + "theoretical_loss": 3.45482722640286, + "tokens_seen": 1838088192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506262626262627, + "loss": 1.5085, + "theoretical_loss": 3.4548167848802525, + "tokens_seen": 1838153728 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045060606060606063, + "loss": 1.4115, + "theoretical_loss": 3.454806343834144, + "tokens_seen": 1838219264 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505858585858586, + "loss": 1.4466, + "theoretical_loss": 3.454795903264496, + "tokens_seen": 1838284800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045056565656565655, + "loss": 1.7046, + "theoretical_loss": 3.4547854631712696, + "tokens_seen": 1838350336 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045054545454545456, + "loss": 1.6425, + "theoretical_loss": 3.454775023554426, + "tokens_seen": 1838415872 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045052525252525257, + "loss": 1.3357, + "theoretical_loss": 3.454764584413927, + "tokens_seen": 1838481408 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.28757330775260925, + "objective/train/avg_token_score": 0.022113921120762825, + "objective/train/docs_used": 1039769, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9949387907981873, + "objective/train/original_loss": 4.7078962326049805, + "objective/train/theoretical_loss": 3.4547541457497326, + "objective/train/tokens_used": 197406176, + "objective/train/unlikelihood_loss": 0.05575425177812576, + "theoretical_loss": 3.4547541457497326, + "tokens_seen": 1838546944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505050505050505, + "loss": 1.4208, + "theoretical_loss": 3.4547541457497326, + "tokens_seen": 1838546944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504848484848485, + "loss": 1.3145, + "theoretical_loss": 3.4547437075618053, + "tokens_seen": 1838612480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504646464646465, + "loss": 1.4106, + "theoretical_loss": 3.454733269850106, + "tokens_seen": 1838678016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045044444444444445, + "loss": 1.3905, + "theoretical_loss": 3.4547228326145953, + "tokens_seen": 1838743552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045042424242424246, + "loss": 1.6694, + "theoretical_loss": 3.454712395855236, + "tokens_seen": 1838809088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504040404040404, + "loss": 1.5321, + "theoretical_loss": 3.454701959571988, + "tokens_seen": 1838874624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503838383838384, + "loss": 1.6927, + "theoretical_loss": 3.4546915237648137, + "tokens_seen": 1838940160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503636363636364, + "loss": 1.3334, + "theoretical_loss": 3.4546810884336736, + "tokens_seen": 1839005696 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045034343434343434, + "loss": 1.4439, + "theoretical_loss": 3.454670653578529, + "tokens_seen": 1839071232 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045032323232323235, + "loss": 1.5006, + "theoretical_loss": 3.4546602191993427, + "tokens_seen": 1839136768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503030303030303, + "loss": 1.4973, + "theoretical_loss": 3.4546497852960742, + "tokens_seen": 1839202304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502828282828283, + "loss": 1.3218, + "theoretical_loss": 3.4546393518686855, + "tokens_seen": 1839267840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502626262626263, + "loss": 1.3238, + "theoretical_loss": 3.4546289189171375, + "tokens_seen": 1839333376 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045024242424242423, + "loss": 1.2593, + "theoretical_loss": 3.454618486441393, + "tokens_seen": 1839398912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045022222222222225, + "loss": 1.3184, + "theoretical_loss": 3.4546080544414117, + "tokens_seen": 1839464448 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502020202020202, + "loss": 1.264, + "theoretical_loss": 3.4545976229171558, + "tokens_seen": 1839529984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501818181818182, + "loss": 1.0797, + "theoretical_loss": 3.454587191868587, + "tokens_seen": 1839595520 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045016161616161617, + "loss": 1.6226, + "theoretical_loss": 3.454576761295665, + "tokens_seen": 1839661056 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501414141414141, + "loss": 1.6046, + "theoretical_loss": 3.454566331198353, + "tokens_seen": 1839726592 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045012121212121214, + "loss": 1.2764, + "theoretical_loss": 3.4545559015766116, + "tokens_seen": 1839792128 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045010101010101015, + "loss": 1.4158, + "theoretical_loss": 3.4545454724304028, + "tokens_seen": 1839857664 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500808080808081, + "loss": 1.5471, + "theoretical_loss": 3.4545350437596865, + "tokens_seen": 1839923200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045006060606060606, + "loss": 1.4801, + "theoretical_loss": 3.4545246155644254, + "tokens_seen": 1839988736 + }, + { + "epoch": 0.11, + "learning_rate": 0.000450040404040404, + "loss": 1.4026, + "theoretical_loss": 3.4545141878445804, + "tokens_seen": 1840054272 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045002020202020203, + "loss": 1.6834, + "theoretical_loss": 3.4545037606001134, + "tokens_seen": 1840119808 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.6422287225723267, + "objective/train/avg_token_score": 0.03135533630847931, + "objective/train/docs_used": 1040587, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.251701593399048, + "objective/train/original_loss": 3.8327317237854004, + "objective/train/theoretical_loss": 3.4544933338309853, + "objective/train/tokens_used": 199044576, + "objective/train/unlikelihood_loss": 0.05906657129526138, + "theoretical_loss": 3.4544933338309853, + "tokens_seen": 1840185344 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045000000000000004, + "loss": 1.7251, + "theoretical_loss": 3.4544933338309853, + "tokens_seen": 1840185344 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449979797979798, + "loss": 1.4279, + "theoretical_loss": 3.4544829075371575, + "tokens_seen": 1840250880 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044995959595959596, + "loss": 1.5526, + "theoretical_loss": 3.4544724817185912, + "tokens_seen": 1840316416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004499393939393939, + "loss": 1.3022, + "theoretical_loss": 3.454462056375249, + "tokens_seen": 1840381952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004499191919191919, + "loss": 1.6946, + "theoretical_loss": 3.454451631507091, + "tokens_seen": 1840447488 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044989898989898993, + "loss": 1.3439, + "theoretical_loss": 3.454441207114079, + "tokens_seen": 1840513024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498787878787879, + "loss": 1.5178, + "theoretical_loss": 3.454430783196174, + "tokens_seen": 1840578560 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044985858585858585, + "loss": 1.328, + "theoretical_loss": 3.4544203597533385, + "tokens_seen": 1840644096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498383838383838, + "loss": 1.3865, + "theoretical_loss": 3.454409936785533, + "tokens_seen": 1840709632 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044981818181818187, + "loss": 1.4214, + "theoretical_loss": 3.45439951429272, + "tokens_seen": 1840775168 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044979797979797983, + "loss": 1.4532, + "theoretical_loss": 3.4543890922748597, + "tokens_seen": 1840840704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497777777777778, + "loss": 1.7572, + "theoretical_loss": 3.454378670731914, + "tokens_seen": 1840906240 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044975757575757574, + "loss": 1.2789, + "theoretical_loss": 3.454368249663845, + "tokens_seen": 1840971776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497373737373737, + "loss": 1.4031, + "theoretical_loss": 3.454357829070613, + "tokens_seen": 1841037312 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044971717171717176, + "loss": 1.2913, + "theoretical_loss": 3.4543474089521804, + "tokens_seen": 1841102848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496969696969697, + "loss": 1.3861, + "theoretical_loss": 3.4543369893085085, + "tokens_seen": 1841168384 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496767676767677, + "loss": 1.2665, + "theoretical_loss": 3.4543265701395582, + "tokens_seen": 1841233920 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044965656565656563, + "loss": 1.4635, + "theoretical_loss": 3.4543161514452914, + "tokens_seen": 1841299456 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044963636363636364, + "loss": 1.3889, + "theoretical_loss": 3.45430573322567, + "tokens_seen": 1841364992 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044961616161616166, + "loss": 1.5038, + "theoretical_loss": 3.4542953154806546, + "tokens_seen": 1841430528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495959595959596, + "loss": 1.4529, + "theoretical_loss": 3.4542848982102075, + "tokens_seen": 1841496064 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044957575757575757, + "loss": 1.3667, + "theoretical_loss": 3.4542744814142896, + "tokens_seen": 1841561600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495555555555555, + "loss": 1.4548, + "theoretical_loss": 3.454264065092863, + "tokens_seen": 1841627136 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044953535353535354, + "loss": 1.4561, + "theoretical_loss": 3.454253649245888, + "tokens_seen": 1841692672 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044951515151515155, + "loss": 1.3452, + "theoretical_loss": 3.4542432338733278, + "tokens_seen": 1841758208 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.3169599175453186, + "objective/train/avg_token_score": 0.005386400036513805, + "objective/train/docs_used": 1041999, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0743252038955688, + "objective/train/original_loss": 4.4719977378845215, + "objective/train/theoretical_loss": 3.4542328189751426, + "objective/train/tokens_used": 200682976, + "objective/train/unlikelihood_loss": 0.08110522478818893, + "theoretical_loss": 3.4542328189751426, + "tokens_seen": 1841823744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494949494949495, + "loss": 1.1803, + "theoretical_loss": 3.4542328189751426, + "tokens_seen": 1841823744 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044947474747474746, + "loss": 1.474, + "theoretical_loss": 3.4542224045512944, + "tokens_seen": 1841889280 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494545454545455, + "loss": 1.5223, + "theoretical_loss": 3.4542119906017446, + "tokens_seen": 1841954816 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044943434343434343, + "loss": 1.4056, + "theoretical_loss": 3.4542015771264554, + "tokens_seen": 1842020352 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044941414141414144, + "loss": 1.4223, + "theoretical_loss": 3.454191164125387, + "tokens_seen": 1842085888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493939393939394, + "loss": 1.4174, + "theoretical_loss": 3.4541807515985017, + "tokens_seen": 1842151424 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044937373737373735, + "loss": 1.5226, + "theoretical_loss": 3.4541703395457617, + "tokens_seen": 1842216960 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044935353535353537, + "loss": 1.4965, + "theoretical_loss": 3.4541599279671273, + "tokens_seen": 1842282496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493333333333333, + "loss": 1.5692, + "theoretical_loss": 3.454149516862561, + "tokens_seen": 1842348032 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044931313131313133, + "loss": 1.3349, + "theoretical_loss": 3.4541391062320237, + "tokens_seen": 1842413568 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492929292929293, + "loss": 1.4055, + "theoretical_loss": 3.454128696075477, + "tokens_seen": 1842479104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492727272727273, + "loss": 1.3759, + "theoretical_loss": 3.454118286392883, + "tokens_seen": 1842544640 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044925252525252526, + "loss": 1.5596, + "theoretical_loss": 3.454107877184203, + "tokens_seen": 1842610176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492323232323232, + "loss": 1.3979, + "theoretical_loss": 3.4540974684493984, + "tokens_seen": 1842675712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492121212121212, + "loss": 1.6526, + "theoretical_loss": 3.4540870601884306, + "tokens_seen": 1842741248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491919191919192, + "loss": 1.5212, + "theoretical_loss": 3.4540766524012616, + "tokens_seen": 1842806784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491717171717172, + "loss": 1.5532, + "theoretical_loss": 3.454066245087853, + "tokens_seen": 1842872320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044915151515151515, + "loss": 1.5621, + "theoretical_loss": 3.4540558382481663, + "tokens_seen": 1842937856 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044913131313131316, + "loss": 1.3275, + "theoretical_loss": 3.4540454318821627, + "tokens_seen": 1843003392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491111111111111, + "loss": 1.4118, + "theoretical_loss": 3.4540350259898043, + "tokens_seen": 1843068928 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044909090909090913, + "loss": 1.5738, + "theoretical_loss": 3.4540246205710528, + "tokens_seen": 1843134464 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490707070707071, + "loss": 1.5702, + "theoretical_loss": 3.454014215625869, + "tokens_seen": 1843200000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044905050505050504, + "loss": 1.5978, + "theoretical_loss": 3.4540038111542155, + "tokens_seen": 1843265536 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044903030303030306, + "loss": 1.4861, + "theoretical_loss": 3.4539934071560534, + "tokens_seen": 1843331072 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449010101010101, + "loss": 1.5306, + "theoretical_loss": 3.453983003631344, + "tokens_seen": 1843396608 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.30321359634399414, + "objective/train/avg_token_score": 0.03322860598564148, + "objective/train/docs_used": 1042423, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0053961277008057, + "objective/train/original_loss": 4.474796772003174, + "objective/train/theoretical_loss": 3.45397260058005, + "objective/train/tokens_used": 202321376, + "objective/train/unlikelihood_loss": 0.05541876330971718, + "theoretical_loss": 3.45397260058005, + "tokens_seen": 1843462144 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448989898989899, + "loss": 1.5138, + "theoretical_loss": 3.45397260058005, + "tokens_seen": 1843462144 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448969696969697, + "loss": 1.5161, + "theoretical_loss": 3.453962198002132, + "tokens_seen": 1843527680 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044894949494949494, + "loss": 1.2524, + "theoretical_loss": 3.4539517958975514, + "tokens_seen": 1843593216 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044892929292929295, + "loss": 1.5945, + "theoretical_loss": 3.4539413942662707, + "tokens_seen": 1843658752 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044890909090909096, + "loss": 1.5995, + "theoretical_loss": 3.4539309931082514, + "tokens_seen": 1843724288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488888888888889, + "loss": 1.4001, + "theoretical_loss": 3.4539205924234553, + "tokens_seen": 1843789824 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044886868686868687, + "loss": 1.5303, + "theoretical_loss": 3.4539101922118434, + "tokens_seen": 1843855360 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044884848484848483, + "loss": 1.3048, + "theoretical_loss": 3.4538997924733774, + "tokens_seen": 1843920896 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044882828282828284, + "loss": 1.4622, + "theoretical_loss": 3.4538893932080197, + "tokens_seen": 1843986432 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044880808080808085, + "loss": 1.3001, + "theoretical_loss": 3.4538789944157315, + "tokens_seen": 1844051968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487878787878788, + "loss": 1.232, + "theoretical_loss": 3.4538685960964743, + "tokens_seen": 1844117504 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044876767676767677, + "loss": 1.5215, + "theoretical_loss": 3.45385819825021, + "tokens_seen": 1844183040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487474747474747, + "loss": 1.4533, + "theoretical_loss": 3.4538478008769, + "tokens_seen": 1844248576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487272727272728, + "loss": 1.2763, + "theoretical_loss": 3.4538374039765065, + "tokens_seen": 1844314112 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044870707070707074, + "loss": 1.6231, + "theoretical_loss": 3.4538270075489907, + "tokens_seen": 1844379648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486868686868687, + "loss": 1.4151, + "theoretical_loss": 3.4538166115943145, + "tokens_seen": 1844445184 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044866666666666666, + "loss": 1.5396, + "theoretical_loss": 3.45380621611244, + "tokens_seen": 1844510720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486464646464646, + "loss": 1.4981, + "theoretical_loss": 3.453795821103328, + "tokens_seen": 1844576256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486262626262627, + "loss": 1.2735, + "theoretical_loss": 3.4537854265669408, + "tokens_seen": 1844641792 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044860606060606064, + "loss": 1.4933, + "theoretical_loss": 3.4537750325032404, + "tokens_seen": 1844707328 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485858585858586, + "loss": 1.3423, + "theoretical_loss": 3.4537646389121877, + "tokens_seen": 1844772864 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044856565656565655, + "loss": 1.4131, + "theoretical_loss": 3.4537542457937453, + "tokens_seen": 1844838400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485454545454545, + "loss": 1.3256, + "theoretical_loss": 3.4537438531478735, + "tokens_seen": 1844903936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485252525252526, + "loss": 1.3916, + "theoretical_loss": 3.4537334609745356, + "tokens_seen": 1844969472 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044850505050505053, + "loss": 1.5918, + "theoretical_loss": 3.4537230692736927, + "tokens_seen": 1845035008 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.3615591526031494, + "objective/train/avg_token_score": 0.012851543724536896, + "objective/train/docs_used": 1043833, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1701569557189941, + "objective/train/original_loss": 4.571152687072754, + "objective/train/theoretical_loss": 3.4537126780453065, + "objective/train/tokens_used": 203959776, + "objective/train/unlikelihood_loss": 0.05402657762169838, + "theoretical_loss": 3.4537126780453065, + "tokens_seen": 1845100544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484848484848485, + "loss": 1.4921, + "theoretical_loss": 3.4537126780453065, + "tokens_seen": 1845100544 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044846464646464644, + "loss": 1.4343, + "theoretical_loss": 3.453702287289339, + "tokens_seen": 1845166080 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044844444444444445, + "loss": 1.4802, + "theoretical_loss": 3.4536918970057515, + "tokens_seen": 1845231616 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044842424242424247, + "loss": 1.4453, + "theoretical_loss": 3.453681507194506, + "tokens_seen": 1845297152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484040404040404, + "loss": 1.6961, + "theoretical_loss": 3.4536711178555644, + "tokens_seen": 1845362688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483838383838384, + "loss": 1.4932, + "theoretical_loss": 3.453660728988888, + "tokens_seen": 1845428224 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044836363636363634, + "loss": 1.4792, + "theoretical_loss": 3.453650340594439, + "tokens_seen": 1845493760 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044834343434343435, + "loss": 1.454, + "theoretical_loss": 3.453639952672179, + "tokens_seen": 1845559296 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044832323232323236, + "loss": 1.5949, + "theoretical_loss": 3.4536295652220694, + "tokens_seen": 1845624832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483030303030303, + "loss": 1.4518, + "theoretical_loss": 3.453619178244073, + "tokens_seen": 1845690368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044828282828282827, + "loss": 1.5919, + "theoretical_loss": 3.4536087917381506, + "tokens_seen": 1845755904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482626262626263, + "loss": 1.6802, + "theoretical_loss": 3.4535984057042644, + "tokens_seen": 1845821440 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044824242424242424, + "loss": 1.3382, + "theoretical_loss": 3.4535880201423765, + "tokens_seen": 1845886976 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044822222222222225, + "loss": 1.5491, + "theoretical_loss": 3.4535776350524476, + "tokens_seen": 1845952512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482020202020202, + "loss": 1.3793, + "theoretical_loss": 3.453567250434441, + "tokens_seen": 1846018048 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044818181818181816, + "loss": 1.5925, + "theoretical_loss": 3.453556866288317, + "tokens_seen": 1846083584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481616161616162, + "loss": 1.5228, + "theoretical_loss": 3.453546482614038, + "tokens_seen": 1846149120 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044814141414141413, + "loss": 1.5559, + "theoretical_loss": 3.4535360994115667, + "tokens_seen": 1846214656 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044812121212121214, + "loss": 1.5943, + "theoretical_loss": 3.453525716680863, + "tokens_seen": 1846280192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481010101010101, + "loss": 1.4601, + "theoretical_loss": 3.453515334421891, + "tokens_seen": 1846345728 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004480808080808081, + "loss": 1.3743, + "theoretical_loss": 3.453504952634611, + "tokens_seen": 1846411264 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044806060606060607, + "loss": 1.4953, + "theoretical_loss": 3.453494571318985, + "tokens_seen": 1846476800 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448040404040404, + "loss": 1.5075, + "theoretical_loss": 3.453484190474976, + "tokens_seen": 1846542336 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044802020202020204, + "loss": 1.4881, + "theoretical_loss": 3.4534738101025435, + "tokens_seen": 1846607872 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448, + "loss": 1.2744, + "theoretical_loss": 3.4534634302016514, + "tokens_seen": 1846673408 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.4943181872367859, + "objective/train/avg_token_score": 0.03060787171125412, + "objective/train/docs_used": 1044447, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6434940099716187, + "objective/train/original_loss": 4.084818363189697, + "objective/train/theoretical_loss": 3.4534530507722607, + "objective/train/tokens_used": 205598176, + "objective/train/unlikelihood_loss": 0.05918062478303909, + "theoretical_loss": 3.4534530507722607, + "tokens_seen": 1846738944 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447979797979798, + "loss": 1.5188, + "theoretical_loss": 3.4534530507722607, + "tokens_seen": 1846738944 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044795959595959596, + "loss": 1.5925, + "theoretical_loss": 3.453442671814334, + "tokens_seen": 1846804480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004479393939393939, + "loss": 1.3562, + "theoretical_loss": 3.4534322933278316, + "tokens_seen": 1846870016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044791919191919193, + "loss": 1.4972, + "theoretical_loss": 3.453421915312717, + "tokens_seen": 1846935552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044789898989898994, + "loss": 1.306, + "theoretical_loss": 3.453411537768951, + "tokens_seen": 1847001088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478787878787879, + "loss": 1.3592, + "theoretical_loss": 3.4534011606964965, + "tokens_seen": 1847066624 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044785858585858585, + "loss": 1.6231, + "theoretical_loss": 3.4533907840953146, + "tokens_seen": 1847132160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478383838383838, + "loss": 1.5106, + "theoretical_loss": 3.453380407965367, + "tokens_seen": 1847197696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478181818181818, + "loss": 1.438, + "theoretical_loss": 3.4533700323066157, + "tokens_seen": 1847263232 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044779797979797983, + "loss": 1.3438, + "theoretical_loss": 3.4533596571190235, + "tokens_seen": 1847328768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477777777777778, + "loss": 1.3831, + "theoretical_loss": 3.4533492824025513, + "tokens_seen": 1847394304 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044775757575757575, + "loss": 1.423, + "theoretical_loss": 3.453338908157161, + "tokens_seen": 1847459840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477373737373737, + "loss": 1.5044, + "theoretical_loss": 3.4533285343828153, + "tokens_seen": 1847525376 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044771717171717177, + "loss": 1.5387, + "theoretical_loss": 3.453318161079475, + "tokens_seen": 1847590912 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476969696969697, + "loss": 1.2799, + "theoretical_loss": 3.453307788247103, + "tokens_seen": 1847656448 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476767676767677, + "loss": 1.5208, + "theoretical_loss": 3.45329741588566, + "tokens_seen": 1847721984 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044765656565656564, + "loss": 1.6915, + "theoretical_loss": 3.4532870439951098, + "tokens_seen": 1847787520 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476363636363636, + "loss": 1.7828, + "theoretical_loss": 3.453276672575413, + "tokens_seen": 1847853056 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044761616161616166, + "loss": 1.515, + "theoretical_loss": 3.4532663016265315, + "tokens_seen": 1847918592 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475959595959596, + "loss": 1.3178, + "theoretical_loss": 3.4532559311484277, + "tokens_seen": 1847984128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475757575757576, + "loss": 1.4015, + "theoretical_loss": 3.4532455611410633, + "tokens_seen": 1848049664 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044755555555555553, + "loss": 1.3752, + "theoretical_loss": 3.4532351916044, + "tokens_seen": 1848115200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044753535353535354, + "loss": 1.4995, + "theoretical_loss": 3.4532248225384006, + "tokens_seen": 1848180736 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044751515151515155, + "loss": 1.2319, + "theoretical_loss": 3.453214453943026, + "tokens_seen": 1848246272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474949494949495, + "loss": 1.3379, + "theoretical_loss": 3.4532040858182387, + "tokens_seen": 1848311808 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.3084677457809448, + "objective/train/avg_token_score": 0.008858302608132362, + "objective/train/docs_used": 1045763, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8898696899414062, + "objective/train/original_loss": 4.509169101715088, + "objective/train/theoretical_loss": 3.453193718164001, + "objective/train/tokens_used": 207236576, + "objective/train/unlikelihood_loss": 0.06134689226746559, + "theoretical_loss": 3.453193718164001, + "tokens_seen": 1848377344 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044747474747474747, + "loss": 1.286, + "theoretical_loss": 3.453193718164001, + "tokens_seen": 1848377344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474545454545454, + "loss": 1.4064, + "theoretical_loss": 3.4531833509802734, + "tokens_seen": 1848442880 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044743434343434344, + "loss": 1.3498, + "theoretical_loss": 3.45317298426702, + "tokens_seen": 1848508416 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044741414141414145, + "loss": 1.1875, + "theoretical_loss": 3.453162618024201, + "tokens_seen": 1848573952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473939393939394, + "loss": 1.5256, + "theoretical_loss": 3.4531522522517797, + "tokens_seen": 1848639488 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044737373737373736, + "loss": 1.5795, + "theoretical_loss": 3.453141886949717, + "tokens_seen": 1848705024 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044735353535353537, + "loss": 1.4829, + "theoretical_loss": 3.453131522117975, + "tokens_seen": 1848770560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473333333333334, + "loss": 1.6749, + "theoretical_loss": 3.4531211577565166, + "tokens_seen": 1848836096 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044731313131313134, + "loss": 1.5497, + "theoretical_loss": 3.453110793865303, + "tokens_seen": 1848901632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472929292929293, + "loss": 1.431, + "theoretical_loss": 3.4531004304442963, + "tokens_seen": 1848967168 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044727272727272725, + "loss": 1.686, + "theoretical_loss": 3.4530900674934584, + "tokens_seen": 1849032704 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044725252525252526, + "loss": 1.3995, + "theoretical_loss": 3.453079705012752, + "tokens_seen": 1849098240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472323232323233, + "loss": 1.5956, + "theoretical_loss": 3.4530693430021384, + "tokens_seen": 1849163776 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044721212121212123, + "loss": 1.6193, + "theoretical_loss": 3.45305898146158, + "tokens_seen": 1849229312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471919191919192, + "loss": 1.3201, + "theoretical_loss": 3.453048620391038, + "tokens_seen": 1849294848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471717171717172, + "loss": 1.2248, + "theoretical_loss": 3.4530382597904756, + "tokens_seen": 1849360384 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044715151515151516, + "loss": 1.2997, + "theoretical_loss": 3.4530278996598547, + "tokens_seen": 1849425920 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044713131313131317, + "loss": 1.3588, + "theoretical_loss": 3.453017539999136, + "tokens_seen": 1849491456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471111111111111, + "loss": 1.5867, + "theoretical_loss": 3.4530071808082834, + "tokens_seen": 1849556992 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470909090909091, + "loss": 1.5271, + "theoretical_loss": 3.4529968220872576, + "tokens_seen": 1849622528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470707070707071, + "loss": 1.3612, + "theoretical_loss": 3.4529864638360213, + "tokens_seen": 1849688064 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044705050505050505, + "loss": 1.7033, + "theoretical_loss": 3.452976106054536, + "tokens_seen": 1849753600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044703030303030306, + "loss": 1.5997, + "theoretical_loss": 3.4529657487427636, + "tokens_seen": 1849819136 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447010101010101, + "loss": 1.2692, + "theoretical_loss": 3.4529553919006672, + "tokens_seen": 1849884672 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044698989898989903, + "loss": 1.4866, + "theoretical_loss": 3.4529450355282085, + "tokens_seen": 1849950208 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.5390396118164062, + "objective/train/avg_token_score": 0.004928861279040575, + "objective/train/docs_used": 1046397, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7157353162765503, + "objective/train/original_loss": 3.4299583435058594, + "objective/train/theoretical_loss": 3.452934679625349, + "objective/train/tokens_used": 208874976, + "objective/train/unlikelihood_loss": 0.08482911437749863, + "theoretical_loss": 3.452934679625349, + "tokens_seen": 1850015744 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446969696969697, + "loss": 1.5823, + "theoretical_loss": 3.452934679625349, + "tokens_seen": 1850015744 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044694949494949494, + "loss": 1.4851, + "theoretical_loss": 3.4529243241920513, + "tokens_seen": 1850081280 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044692929292929295, + "loss": 1.5261, + "theoretical_loss": 3.452913969228277, + "tokens_seen": 1850146816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469090909090909, + "loss": 1.4711, + "theoretical_loss": 3.4529036147339887, + "tokens_seen": 1850212352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468888888888889, + "loss": 1.5649, + "theoretical_loss": 3.452893260709148, + "tokens_seen": 1850277888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468686868686869, + "loss": 1.5514, + "theoretical_loss": 3.4528829071537177, + "tokens_seen": 1850343424 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044684848484848484, + "loss": 1.3146, + "theoretical_loss": 3.4528725540676595, + "tokens_seen": 1850408960 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044682828282828285, + "loss": 1.5784, + "theoretical_loss": 3.452862201450935, + "tokens_seen": 1850474496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468080808080808, + "loss": 1.7336, + "theoretical_loss": 3.452851849303507, + "tokens_seen": 1850540032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467878787878788, + "loss": 1.4407, + "theoretical_loss": 3.452841497625337, + "tokens_seen": 1850605568 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044676767676767677, + "loss": 1.1921, + "theoretical_loss": 3.452831146416388, + "tokens_seen": 1850671104 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044674747474747473, + "loss": 1.4424, + "theoretical_loss": 3.4528207956766215, + "tokens_seen": 1850736640 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044672727272727274, + "loss": 1.6409, + "theoretical_loss": 3.4528104454059996, + "tokens_seen": 1850802176 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044670707070707075, + "loss": 1.4714, + "theoretical_loss": 3.4528000956044846, + "tokens_seen": 1850867712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466868686868687, + "loss": 1.3572, + "theoretical_loss": 3.4527897462720385, + "tokens_seen": 1850933248 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044666666666666666, + "loss": 1.2197, + "theoretical_loss": 3.452779397408624, + "tokens_seen": 1850998784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466464646464646, + "loss": 1.4201, + "theoretical_loss": 3.452769049014202, + "tokens_seen": 1851064320 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044662626262626263, + "loss": 1.4719, + "theoretical_loss": 3.4527587010887357, + "tokens_seen": 1851129856 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044660606060606064, + "loss": 1.4856, + "theoretical_loss": 3.4527483536321864, + "tokens_seen": 1851195392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465858585858586, + "loss": 1.5586, + "theoretical_loss": 3.4527380066445175, + "tokens_seen": 1851260928 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044656565656565656, + "loss": 1.4921, + "theoretical_loss": 3.4527276601256904, + "tokens_seen": 1851326464 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465454545454545, + "loss": 1.3876, + "theoretical_loss": 3.4527173140756666, + "tokens_seen": 1851392000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465252525252526, + "loss": 1.6662, + "theoretical_loss": 3.4527069684944096, + "tokens_seen": 1851457536 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044650505050505054, + "loss": 1.3775, + "theoretical_loss": 3.4526966233818808, + "tokens_seen": 1851523072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464848484848485, + "loss": 1.3225, + "theoretical_loss": 3.4526862787380423, + "tokens_seen": 1851588608 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.2721163332462311, + "objective/train/avg_token_score": 0.026380645111203194, + "objective/train/docs_used": 1047383, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.913923442363739, + "objective/train/original_loss": 4.239612102508545, + "objective/train/theoretical_loss": 3.4526759345628566, + "objective/train/tokens_used": 210513376, + "objective/train/unlikelihood_loss": 0.079360231757164, + "theoretical_loss": 3.4526759345628566, + "tokens_seen": 1851654144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044646464646464645, + "loss": 1.4303, + "theoretical_loss": 3.4526759345628566, + "tokens_seen": 1851654144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464444444444444, + "loss": 1.2667, + "theoretical_loss": 3.452665590856286, + "tokens_seen": 1851719680 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044642424242424247, + "loss": 1.4178, + "theoretical_loss": 3.452655247618292, + "tokens_seen": 1851785216 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044640404040404043, + "loss": 1.3794, + "theoretical_loss": 3.452644904848838, + "tokens_seen": 1851850752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463838383838384, + "loss": 1.5685, + "theoretical_loss": 3.4526345625478845, + "tokens_seen": 1851916288 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044636363636363634, + "loss": 1.553, + "theoretical_loss": 3.4526242207153954, + "tokens_seen": 1851981824 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044634343434343435, + "loss": 1.7237, + "theoretical_loss": 3.4526138793513317, + "tokens_seen": 1852047360 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044632323232323236, + "loss": 1.753, + "theoretical_loss": 3.452603538455656, + "tokens_seen": 1852112896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463030303030303, + "loss": 1.4571, + "theoretical_loss": 3.452593198028331, + "tokens_seen": 1852178432 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462828282828283, + "loss": 1.31, + "theoretical_loss": 3.4525828580693183, + "tokens_seen": 1852243968 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044626262626262623, + "loss": 1.5186, + "theoretical_loss": 3.45257251857858, + "tokens_seen": 1852309504 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044624242424242425, + "loss": 1.6351, + "theoretical_loss": 3.452562179556079, + "tokens_seen": 1852375040 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044622222222222226, + "loss": 1.1539, + "theoretical_loss": 3.452551841001777, + "tokens_seen": 1852440576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462020202020202, + "loss": 1.3132, + "theoretical_loss": 3.4525415029156363, + "tokens_seen": 1852506112 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044618181818181817, + "loss": 1.2812, + "theoretical_loss": 3.452531165297619, + "tokens_seen": 1852571648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461616161616162, + "loss": 1.606, + "theoretical_loss": 3.452520828147688, + "tokens_seen": 1852637184 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044614141414141414, + "loss": 1.4788, + "theoretical_loss": 3.452510491465805, + "tokens_seen": 1852702720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044612121212121215, + "loss": 1.5314, + "theoretical_loss": 3.4525001552519323, + "tokens_seen": 1852768256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461010101010101, + "loss": 1.309, + "theoretical_loss": 3.4524898195060323, + "tokens_seen": 1852833792 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044608080808080806, + "loss": 1.384, + "theoretical_loss": 3.4524794842280673, + "tokens_seen": 1852899328 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460606060606061, + "loss": 1.462, + "theoretical_loss": 3.4524691494179995, + "tokens_seen": 1852964864 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044604040404040403, + "loss": 1.6896, + "theoretical_loss": 3.452458815075791, + "tokens_seen": 1853030400 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044602020202020204, + "loss": 1.4372, + "theoretical_loss": 3.452448481201404, + "tokens_seen": 1853095936 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446, + "loss": 1.4209, + "theoretical_loss": 3.452438147794801, + "tokens_seen": 1853161472 + }, + { + "epoch": 0.12, + "learning_rate": 0.000445979797979798, + "loss": 1.422, + "theoretical_loss": 3.4524278148559446, + "tokens_seen": 1853227008 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.31262218952178955, + "objective/train/avg_token_score": 0.019027452915906906, + "objective/train/docs_used": 1048172, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1232824325561523, + "objective/train/original_loss": 4.744579792022705, + "objective/train/theoretical_loss": 3.4524174823847966, + "objective/train/tokens_used": 212151776, + "objective/train/unlikelihood_loss": 0.04645653814077377, + "theoretical_loss": 3.4524174823847966, + "tokens_seen": 1853292544 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044595959595959597, + "loss": 1.1746, + "theoretical_loss": 3.4524174823847966, + "tokens_seen": 1853292544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459393939393939, + "loss": 1.4012, + "theoretical_loss": 3.452407150381319, + "tokens_seen": 1853358080 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044591919191919193, + "loss": 1.4507, + "theoretical_loss": 3.4523968188454752, + "tokens_seen": 1853423616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458989898989899, + "loss": 1.463, + "theoretical_loss": 3.452386487777227, + "tokens_seen": 1853489152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458787878787879, + "loss": 1.4316, + "theoretical_loss": 3.452376157176536, + "tokens_seen": 1853554688 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044585858585858586, + "loss": 1.7792, + "theoretical_loss": 3.452365827043365, + "tokens_seen": 1853620224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458383838383838, + "loss": 1.6054, + "theoretical_loss": 3.4523554973776767, + "tokens_seen": 1853685760 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044581818181818183, + "loss": 1.3471, + "theoretical_loss": 3.452345168179433, + "tokens_seen": 1853751296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044579797979797984, + "loss": 1.4412, + "theoretical_loss": 3.4523348394485964, + "tokens_seen": 1853816832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457777777777778, + "loss": 1.3501, + "theoretical_loss": 3.452324511185129, + "tokens_seen": 1853882368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044575757575757575, + "loss": 1.5304, + "theoretical_loss": 3.452314183388993, + "tokens_seen": 1853947904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457373737373737, + "loss": 1.4022, + "theoretical_loss": 3.4523038560601513, + "tokens_seen": 1854013440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457171717171717, + "loss": 1.3444, + "theoretical_loss": 3.452293529198566, + "tokens_seen": 1854078976 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044569696969696973, + "loss": 1.2717, + "theoretical_loss": 3.4522832028041988, + "tokens_seen": 1854144512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456767676767677, + "loss": 1.3764, + "theoretical_loss": 3.4522728768770135, + "tokens_seen": 1854210048 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044565656565656564, + "loss": 1.3803, + "theoretical_loss": 3.452262551416971, + "tokens_seen": 1854275584 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044563636363636366, + "loss": 1.4479, + "theoretical_loss": 3.4522522264240347, + "tokens_seen": 1854341120 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044561616161616167, + "loss": 1.6955, + "theoretical_loss": 3.452241901898166, + "tokens_seen": 1854406656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455959595959596, + "loss": 1.6218, + "theoretical_loss": 3.452231577839328, + "tokens_seen": 1854472192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455757575757576, + "loss": 1.3389, + "theoretical_loss": 3.452221254247483, + "tokens_seen": 1854537728 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044555555555555554, + "loss": 1.521, + "theoretical_loss": 3.4522109311225933, + "tokens_seen": 1854603264 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044553535353535355, + "loss": 1.2394, + "theoretical_loss": 3.4522006084646204, + "tokens_seen": 1854668800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044551515151515156, + "loss": 1.4995, + "theoretical_loss": 3.4521902862735283, + "tokens_seen": 1854734336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454949494949495, + "loss": 1.3931, + "theoretical_loss": 3.4521799645492783, + "tokens_seen": 1854799872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454747474747475, + "loss": 1.5762, + "theoretical_loss": 3.452169643291833, + "tokens_seen": 1854865408 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.40560850501060486, + "objective/train/avg_token_score": 0.008102362044155598, + "objective/train/docs_used": 1049323, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3408201932907104, + "objective/train/original_loss": 4.186563968658447, + "objective/train/theoretical_loss": 3.452159322501155, + "objective/train/tokens_used": 213790176, + "objective/train/unlikelihood_loss": 0.057345665991306305, + "theoretical_loss": 3.452159322501155, + "tokens_seen": 1854930944 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044545454545454543, + "loss": 1.5162, + "theoretical_loss": 3.452159322501155, + "tokens_seen": 1854930944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454343434343435, + "loss": 1.5745, + "theoretical_loss": 3.4521490021772063, + "tokens_seen": 1854996480 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044541414141414145, + "loss": 1.3886, + "theoretical_loss": 3.45213868231995, + "tokens_seen": 1855062016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453939393939394, + "loss": 1.381, + "theoretical_loss": 3.452128362929348, + "tokens_seen": 1855127552 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044537373737373737, + "loss": 1.703, + "theoretical_loss": 3.4521180440053625, + "tokens_seen": 1855193088 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453535353535353, + "loss": 1.569, + "theoretical_loss": 3.452107725547956, + "tokens_seen": 1855258624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453333333333334, + "loss": 1.2288, + "theoretical_loss": 3.4520974075570914, + "tokens_seen": 1855324160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044531313131313135, + "loss": 1.5825, + "theoretical_loss": 3.452087090032731, + "tokens_seen": 1855389696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004452929292929293, + "loss": 1.5339, + "theoretical_loss": 3.452076772974837, + "tokens_seen": 1855455232 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044527272727272726, + "loss": 1.173, + "theoretical_loss": 3.4520664563833723, + "tokens_seen": 1855520768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004452525252525252, + "loss": 1.5338, + "theoretical_loss": 3.452056140258298, + "tokens_seen": 1855586304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004452323232323233, + "loss": 1.2947, + "theoretical_loss": 3.4520458245995784, + "tokens_seen": 1855651840 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044521212121212124, + "loss": 1.4635, + "theoretical_loss": 3.4520355094071746, + "tokens_seen": 1855717376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451919191919192, + "loss": 1.2195, + "theoretical_loss": 3.45202519468105, + "tokens_seen": 1855782912 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044517171717171715, + "loss": 1.5204, + "theoretical_loss": 3.452014880421166, + "tokens_seen": 1855848448 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044515151515151516, + "loss": 1.2559, + "theoretical_loss": 3.452004566627486, + "tokens_seen": 1855913984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451313131313132, + "loss": 1.3033, + "theoretical_loss": 3.4519942532999712, + "tokens_seen": 1855979520 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044511111111111113, + "loss": 1.5715, + "theoretical_loss": 3.451983940438586, + "tokens_seen": 1856045056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450909090909091, + "loss": 1.506, + "theoretical_loss": 3.4519736280432918, + "tokens_seen": 1856110592 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044507070707070704, + "loss": 1.4881, + "theoretical_loss": 3.4519633161140506, + "tokens_seen": 1856176128 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044505050505050506, + "loss": 1.3911, + "theoretical_loss": 3.4519530046508256, + "tokens_seen": 1856241664 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044503030303030307, + "loss": 1.3362, + "theoretical_loss": 3.451942693653579, + "tokens_seen": 1856307200 + }, + { + "epoch": 0.12, + "learning_rate": 0.000445010101010101, + "loss": 1.4433, + "theoretical_loss": 3.4519323831222737, + "tokens_seen": 1856372736 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444989898989899, + "loss": 1.2808, + "theoretical_loss": 3.4519220730568714, + "tokens_seen": 1856438272 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444969696969697, + "loss": 1.43, + "theoretical_loss": 3.4519117634573355, + "tokens_seen": 1856503808 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.33345553278923035, + "objective/train/avg_token_score": 0.06058519333600998, + "objective/train/docs_used": 1049929, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.12827730178833, + "objective/train/original_loss": 4.973564624786377, + "objective/train/theoretical_loss": 3.451901454323628, + "objective/train/tokens_used": 215428576, + "objective/train/unlikelihood_loss": 0.046354275196790695, + "theoretical_loss": 3.451901454323628, + "tokens_seen": 1856569344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044494949494949495, + "loss": 1.4263, + "theoretical_loss": 3.451901454323628, + "tokens_seen": 1856569344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044492929292929296, + "loss": 1.2963, + "theoretical_loss": 3.4518911456557113, + "tokens_seen": 1856634880 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449090909090909, + "loss": 1.6288, + "theoretical_loss": 3.4518808374535483, + "tokens_seen": 1856700416 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044488888888888887, + "loss": 1.3875, + "theoretical_loss": 3.4518705297171013, + "tokens_seen": 1856765952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448686868686869, + "loss": 1.3577, + "theoretical_loss": 3.4518602224463324, + "tokens_seen": 1856831488 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044484848484848484, + "loss": 1.5654, + "theoretical_loss": 3.4518499156412052, + "tokens_seen": 1856897024 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044482828282828285, + "loss": 1.2628, + "theoretical_loss": 3.4518396093016817, + "tokens_seen": 1856962560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448080808080808, + "loss": 1.3431, + "theoretical_loss": 3.451829303427724, + "tokens_seen": 1857028096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004447878787878788, + "loss": 1.4391, + "theoretical_loss": 3.451818998019295, + "tokens_seen": 1857093632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004447676767676768, + "loss": 1.4135, + "theoretical_loss": 3.451808693076357, + "tokens_seen": 1857159168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044474747474747473, + "loss": 1.5886, + "theoretical_loss": 3.4517983885988732, + "tokens_seen": 1857224704 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044472727272727274, + "loss": 1.4285, + "theoretical_loss": 3.4517880845868056, + "tokens_seen": 1857290240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004447070707070707, + "loss": 1.4832, + "theoretical_loss": 3.4517777810401165, + "tokens_seen": 1857355776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446868686868687, + "loss": 1.3972, + "theoretical_loss": 3.4517674779587693, + "tokens_seen": 1857421312 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044466666666666667, + "loss": 1.4377, + "theoretical_loss": 3.451757175342726, + "tokens_seen": 1857486848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446464646464646, + "loss": 1.5561, + "theoretical_loss": 3.451746873191949, + "tokens_seen": 1857552384 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044462626262626264, + "loss": 1.3046, + "theoretical_loss": 3.4517365715064017, + "tokens_seen": 1857617920 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044460606060606065, + "loss": 1.5743, + "theoretical_loss": 3.451726270286046, + "tokens_seen": 1857683456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445858585858586, + "loss": 1.238, + "theoretical_loss": 3.451715969530844, + "tokens_seen": 1857748992 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044456565656565656, + "loss": 1.6605, + "theoretical_loss": 3.4517056692407597, + "tokens_seen": 1857814528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445454545454545, + "loss": 1.5474, + "theoretical_loss": 3.451695369415755, + "tokens_seen": 1857880064 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044452525252525253, + "loss": 1.4325, + "theoretical_loss": 3.4516850700557917, + "tokens_seen": 1857945600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044450505050505054, + "loss": 1.4822, + "theoretical_loss": 3.451674771160833, + "tokens_seen": 1858011136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444848484848485, + "loss": 1.3059, + "theoretical_loss": 3.451664472730842, + "tokens_seen": 1858076672 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044446464646464645, + "loss": 1.3968, + "theoretical_loss": 3.451654174765781, + "tokens_seen": 1858142208 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.40243157744407654, + "objective/train/avg_token_score": 0.01305119413882494, + "objective/train/docs_used": 1050597, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3890475034713745, + "objective/train/original_loss": 4.159512042999268, + "objective/train/theoretical_loss": 3.451643877265613, + "objective/train/tokens_used": 217066976, + "objective/train/unlikelihood_loss": 0.06591649353504181, + "theoretical_loss": 3.451643877265613, + "tokens_seen": 1858207744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444444444444444, + "loss": 1.5441, + "theoretical_loss": 3.451643877265613, + "tokens_seen": 1858207744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444242424242425, + "loss": 1.4623, + "theoretical_loss": 3.451633580230299, + "tokens_seen": 1858273280 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044440404040404043, + "loss": 1.5315, + "theoretical_loss": 3.4516232836598033, + "tokens_seen": 1858338816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443838383838384, + "loss": 1.2582, + "theoretical_loss": 3.451612987554088, + "tokens_seen": 1858404352 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044436363636363635, + "loss": 1.5215, + "theoretical_loss": 3.451602691913116, + "tokens_seen": 1858469888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443434343434343, + "loss": 1.3035, + "theoretical_loss": 3.4515923967368494, + "tokens_seen": 1858535424 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044432323232323237, + "loss": 1.342, + "theoretical_loss": 3.4515821020252506, + "tokens_seen": 1858600960 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443030303030303, + "loss": 1.3374, + "theoretical_loss": 3.4515718077782838, + "tokens_seen": 1858666496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004442828282828283, + "loss": 1.3353, + "theoretical_loss": 3.4515615139959097, + "tokens_seen": 1858732032 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044426262626262624, + "loss": 1.4248, + "theoretical_loss": 3.451551220678092, + "tokens_seen": 1858797568 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044424242424242425, + "loss": 1.47, + "theoretical_loss": 3.4515409278247935, + "tokens_seen": 1858863104 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044422222222222226, + "loss": 1.1401, + "theoretical_loss": 3.4515306354359767, + "tokens_seen": 1858928640 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004442020202020202, + "loss": 1.5552, + "theoretical_loss": 3.4515203435116035, + "tokens_seen": 1858994176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441818181818182, + "loss": 1.4739, + "theoretical_loss": 3.4515100520516375, + "tokens_seen": 1859059712 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044416161616161613, + "loss": 1.4808, + "theoretical_loss": 3.4514997610560414, + "tokens_seen": 1859125248 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044414141414141414, + "loss": 1.3743, + "theoretical_loss": 3.451489470524777, + "tokens_seen": 1859190784 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044412121212121215, + "loss": 1.4206, + "theoretical_loss": 3.4514791804578078, + "tokens_seen": 1859256320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441010101010101, + "loss": 1.5322, + "theoretical_loss": 3.4514688908550966, + "tokens_seen": 1859321856 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044408080808080807, + "loss": 1.4228, + "theoretical_loss": 3.451458601716605, + "tokens_seen": 1859387392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440606060606061, + "loss": 1.7084, + "theoretical_loss": 3.451448313042297, + "tokens_seen": 1859452928 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044404040404040404, + "loss": 1.1568, + "theoretical_loss": 3.451438024832134, + "tokens_seen": 1859518464 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044402020202020205, + "loss": 1.3594, + "theoretical_loss": 3.45142773708608, + "tokens_seen": 1859584000 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444, + "loss": 1.5296, + "theoretical_loss": 3.451417449804097, + "tokens_seen": 1859649536 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044397979797979796, + "loss": 1.243, + "theoretical_loss": 3.451407162986148, + "tokens_seen": 1859715072 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044395959595959597, + "loss": 1.3292, + "theoretical_loss": 3.451396876632195, + "tokens_seen": 1859780608 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.3087732195854187, + "objective/train/avg_token_score": 0.015457415953278542, + "objective/train/docs_used": 1051522, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0185426473617554, + "objective/train/original_loss": 4.571277618408203, + "objective/train/theoretical_loss": 3.4513865907422017, + "objective/train/tokens_used": 218705376, + "objective/train/unlikelihood_loss": 0.05881958827376366, + "theoretical_loss": 3.4513865907422017, + "tokens_seen": 1859846144 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443939393939394, + "loss": 1.3964, + "theoretical_loss": 3.4513865907422017, + "tokens_seen": 1859846144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044391919191919194, + "loss": 1.4011, + "theoretical_loss": 3.4513763053161304, + "tokens_seen": 1859911680 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438989898989899, + "loss": 1.6454, + "theoretical_loss": 3.4513660203539436, + "tokens_seen": 1859977216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438787878787879, + "loss": 1.4499, + "theoretical_loss": 3.4513557358556044, + "tokens_seen": 1860042752 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044385858585858587, + "loss": 1.4571, + "theoretical_loss": 3.4513454518210755, + "tokens_seen": 1860108288 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438383838383839, + "loss": 1.1652, + "theoretical_loss": 3.4513351682503193, + "tokens_seen": 1860173824 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044381818181818183, + "loss": 1.4164, + "theoretical_loss": 3.4513248851432987, + "tokens_seen": 1860239360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437979797979798, + "loss": 1.2335, + "theoretical_loss": 3.451314602499977, + "tokens_seen": 1860304896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437777777777778, + "loss": 1.4228, + "theoretical_loss": 3.451304320320316, + "tokens_seen": 1860370432 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044375757575757576, + "loss": 1.3746, + "theoretical_loss": 3.451294038604279, + "tokens_seen": 1860435968 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044373737373737377, + "loss": 1.6736, + "theoretical_loss": 3.4512837573518294, + "tokens_seen": 1860501504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437171717171717, + "loss": 1.1772, + "theoretical_loss": 3.4512734765629283, + "tokens_seen": 1860567040 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044369696969696974, + "loss": 1.406, + "theoretical_loss": 3.45126319623754, + "tokens_seen": 1860632576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436767676767677, + "loss": 1.4388, + "theoretical_loss": 3.451252916375627, + "tokens_seen": 1860698112 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044365656565656565, + "loss": 1.6811, + "theoretical_loss": 3.4512426369771516, + "tokens_seen": 1860763648 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044363636363636366, + "loss": 1.3693, + "theoretical_loss": 3.4512323580420765, + "tokens_seen": 1860829184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436161616161616, + "loss": 1.4087, + "theoretical_loss": 3.451222079570365, + "tokens_seen": 1860894720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044359595959595963, + "loss": 1.3867, + "theoretical_loss": 3.4512118015619797, + "tokens_seen": 1860960256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435757575757576, + "loss": 1.566, + "theoretical_loss": 3.4512015240168834, + "tokens_seen": 1861025792 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044355555555555554, + "loss": 1.513, + "theoretical_loss": 3.451191246935039, + "tokens_seen": 1861091328 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044353535353535355, + "loss": 1.492, + "theoretical_loss": 3.4511809703164094, + "tokens_seen": 1861156864 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435151515151515, + "loss": 1.5802, + "theoretical_loss": 3.4511706941609566, + "tokens_seen": 1861222400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434949494949495, + "loss": 1.3848, + "theoretical_loss": 3.4511604184686444, + "tokens_seen": 1861287936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434747474747475, + "loss": 1.4916, + "theoretical_loss": 3.4511501432394356, + "tokens_seen": 1861353472 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044345454545454544, + "loss": 1.1949, + "theoretical_loss": 3.451139868473292, + "tokens_seen": 1861419008 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.37634408473968506, + "objective/train/avg_token_score": 0.01542601641267538, + "objective/train/docs_used": 1052031, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.181724190711975, + "objective/train/original_loss": 4.738986968994141, + "objective/train/theoretical_loss": 3.4511295941701774, + "objective/train/tokens_used": 220343776, + "objective/train/unlikelihood_loss": 0.061755482107400894, + "theoretical_loss": 3.4511295941701774, + "tokens_seen": 1861484544 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044343434343434345, + "loss": 1.2778, + "theoretical_loss": 3.4511295941701774, + "tokens_seen": 1861484544 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044341414141414146, + "loss": 1.4889, + "theoretical_loss": 3.4511193203300543, + "tokens_seen": 1861550080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004433939393939394, + "loss": 1.6773, + "theoretical_loss": 3.451109046952886, + "tokens_seen": 1861615616 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044337373737373737, + "loss": 1.4959, + "theoretical_loss": 3.451098774038634, + "tokens_seen": 1861681152 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044335353535353533, + "loss": 1.5357, + "theoretical_loss": 3.4510885015872628, + "tokens_seen": 1861746688 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044333333333333334, + "loss": 1.1331, + "theoretical_loss": 3.4510782295987346, + "tokens_seen": 1861812224 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044331313131313135, + "loss": 1.1775, + "theoretical_loss": 3.4510679580730117, + "tokens_seen": 1861877760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432929292929293, + "loss": 1.4438, + "theoretical_loss": 3.4510576870100573, + "tokens_seen": 1861943296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044327272727272726, + "loss": 1.2656, + "theoretical_loss": 3.4510474164098346, + "tokens_seen": 1862008832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432525252525252, + "loss": 1.3183, + "theoretical_loss": 3.4510371462723066, + "tokens_seen": 1862074368 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432323232323233, + "loss": 1.4267, + "theoretical_loss": 3.4510268765974357, + "tokens_seen": 1862139904 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044321212121212124, + "loss": 1.3859, + "theoretical_loss": 3.451016607385185, + "tokens_seen": 1862205440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431919191919192, + "loss": 1.2745, + "theoretical_loss": 3.451006338635517, + "tokens_seen": 1862270976 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044317171717171716, + "loss": 1.264, + "theoretical_loss": 3.4509960703483946, + "tokens_seen": 1862336512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431515151515151, + "loss": 1.4957, + "theoretical_loss": 3.450985802523782, + "tokens_seen": 1862402048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431313131313132, + "loss": 1.1714, + "theoretical_loss": 3.45097553516164, + "tokens_seen": 1862467584 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044311111111111114, + "loss": 1.6347, + "theoretical_loss": 3.4509652682619327, + "tokens_seen": 1862533120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004430909090909091, + "loss": 1.456, + "theoretical_loss": 3.4509550018246236, + "tokens_seen": 1862598656 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044307070707070705, + "loss": 1.48, + "theoretical_loss": 3.4509447358496743, + "tokens_seen": 1862664192 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044305050505050506, + "loss": 1.5629, + "theoretical_loss": 3.450934470337048, + "tokens_seen": 1862729728 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044303030303030307, + "loss": 1.2863, + "theoretical_loss": 3.450924205286708, + "tokens_seen": 1862795264 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044301010101010103, + "loss": 1.5154, + "theoretical_loss": 3.4509139406986176, + "tokens_seen": 1862860800 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442989898989899, + "loss": 1.4515, + "theoretical_loss": 3.450903676572739, + "tokens_seen": 1862926336 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044296969696969694, + "loss": 1.3576, + "theoretical_loss": 3.4508934129090356, + "tokens_seen": 1862991872 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044294949494949495, + "loss": 1.4711, + "theoretical_loss": 3.4508831497074697, + "tokens_seen": 1863057408 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.35129520297050476, + "objective/train/avg_token_score": 0.03377072885632515, + "objective/train/docs_used": 1053498, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3309406042099, + "objective/train/original_loss": 4.976852893829346, + "objective/train/theoretical_loss": 3.4508728869680043, + "objective/train/tokens_used": 221982176, + "objective/train/unlikelihood_loss": 0.04581369832158089, + "theoretical_loss": 3.4508728869680043, + "tokens_seen": 1863122944 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044292929292929296, + "loss": 1.6527, + "theoretical_loss": 3.4508728869680043, + "tokens_seen": 1863122944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429090909090909, + "loss": 1.2831, + "theoretical_loss": 3.450862624690603, + "tokens_seen": 1863188480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428888888888889, + "loss": 1.2713, + "theoretical_loss": 3.450852362875229, + "tokens_seen": 1863254016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428686868686869, + "loss": 1.5471, + "theoretical_loss": 3.450842101521844, + "tokens_seen": 1863319552 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044284848484848485, + "loss": 1.3051, + "theoretical_loss": 3.4508318406304115, + "tokens_seen": 1863385088 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044282828282828286, + "loss": 1.401, + "theoretical_loss": 3.4508215802008952, + "tokens_seen": 1863450624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428080808080808, + "loss": 1.2803, + "theoretical_loss": 3.450811320233257, + "tokens_seen": 1863516160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044278787878787877, + "loss": 1.3002, + "theoretical_loss": 3.4508010607274606, + "tokens_seen": 1863581696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427676767676768, + "loss": 1.478, + "theoretical_loss": 3.4507908016834685, + "tokens_seen": 1863647232 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044274747474747474, + "loss": 1.4375, + "theoretical_loss": 3.4507805431012435, + "tokens_seen": 1863712768 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044272727272727275, + "loss": 1.3306, + "theoretical_loss": 3.4507702849807496, + "tokens_seen": 1863778304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427070707070707, + "loss": 1.4733, + "theoretical_loss": 3.450760027321948, + "tokens_seen": 1863843840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426868686868687, + "loss": 1.5835, + "theoretical_loss": 3.450749770124804, + "tokens_seen": 1863909376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426666666666667, + "loss": 1.4993, + "theoretical_loss": 3.4507395133892786, + "tokens_seen": 1863974912 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044264646464646463, + "loss": 1.423, + "theoretical_loss": 3.450729257115336, + "tokens_seen": 1864040448 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044262626262626264, + "loss": 1.493, + "theoretical_loss": 3.4507190013029385, + "tokens_seen": 1864105984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426060606060606, + "loss": 1.5426, + "theoretical_loss": 3.45070874595205, + "tokens_seen": 1864171520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425858585858586, + "loss": 1.5896, + "theoretical_loss": 3.4506984910626324, + "tokens_seen": 1864237056 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044256565656565657, + "loss": 1.5533, + "theoretical_loss": 3.450688236634649, + "tokens_seen": 1864302592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425454545454545, + "loss": 1.2419, + "theoretical_loss": 3.450677982668063, + "tokens_seen": 1864368128 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044252525252525254, + "loss": 1.2114, + "theoretical_loss": 3.450667729162838, + "tokens_seen": 1864433664 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044250505050505055, + "loss": 1.3821, + "theoretical_loss": 3.450657476118936, + "tokens_seen": 1864499200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424848484848485, + "loss": 1.4739, + "theoretical_loss": 3.450647223536321, + "tokens_seen": 1864564736 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044246464646464646, + "loss": 1.5295, + "theoretical_loss": 3.450636971414955, + "tokens_seen": 1864630272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424444444444444, + "loss": 1.2789, + "theoretical_loss": 3.4506267197548017, + "tokens_seen": 1864695808 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.39766618609428406, + "objective/train/avg_token_score": 0.03243307024240494, + "objective/train/docs_used": 1054143, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4144164323806763, + "objective/train/original_loss": 5.110235691070557, + "objective/train/theoretical_loss": 3.4506164685558245, + "objective/train/tokens_used": 223620576, + "objective/train/unlikelihood_loss": 0.04839473217725754, + "theoretical_loss": 3.4506164685558245, + "tokens_seen": 1864761344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044242424242424243, + "loss": 1.4935, + "theoretical_loss": 3.4506164685558245, + "tokens_seen": 1864761344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044240404040404044, + "loss": 1.3372, + "theoretical_loss": 3.4506062178179855, + "tokens_seen": 1864826880 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423838383838384, + "loss": 1.4895, + "theoretical_loss": 3.450595967541248, + "tokens_seen": 1864892416 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044236363636363635, + "loss": 1.3013, + "theoretical_loss": 3.450585717725576, + "tokens_seen": 1864957952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423434343434343, + "loss": 1.5954, + "theoretical_loss": 3.450575468370931, + "tokens_seen": 1865023488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423232323232324, + "loss": 1.5314, + "theoretical_loss": 3.4505652194772773, + "tokens_seen": 1865089024 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044230303030303033, + "loss": 1.386, + "theoretical_loss": 3.4505549710445775, + "tokens_seen": 1865154560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422828282828283, + "loss": 1.3209, + "theoretical_loss": 3.450544723072795, + "tokens_seen": 1865220096 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044226262626262625, + "loss": 1.5006, + "theoretical_loss": 3.4505344755618923, + "tokens_seen": 1865285632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422424242424242, + "loss": 1.6062, + "theoretical_loss": 3.450524228511833, + "tokens_seen": 1865351168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044222222222222227, + "loss": 1.4058, + "theoretical_loss": 3.4505139819225796, + "tokens_seen": 1865416704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422020202020202, + "loss": 1.5056, + "theoretical_loss": 3.450503735794096, + "tokens_seen": 1865482240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421818181818182, + "loss": 1.4215, + "theoretical_loss": 3.4504934901263447, + "tokens_seen": 1865547776 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044216161616161614, + "loss": 1.2773, + "theoretical_loss": 3.450483244919289, + "tokens_seen": 1865613312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421414141414142, + "loss": 1.7203, + "theoretical_loss": 3.450473000172892, + "tokens_seen": 1865678848 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044212121212121216, + "loss": 1.412, + "theoretical_loss": 3.450462755887117, + "tokens_seen": 1865744384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421010101010101, + "loss": 1.3453, + "theoretical_loss": 3.4504525120619265, + "tokens_seen": 1865809920 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420808080808081, + "loss": 1.4669, + "theoretical_loss": 3.4504422686972838, + "tokens_seen": 1865875456 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044206060606060603, + "loss": 1.4215, + "theoretical_loss": 3.4504320257931527, + "tokens_seen": 1865940992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420404040404041, + "loss": 1.5517, + "theoretical_loss": 3.450421783349496, + "tokens_seen": 1866006528 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044202020202020205, + "loss": 1.2322, + "theoretical_loss": 3.450411541366276, + "tokens_seen": 1866072064 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442, + "loss": 1.578, + "theoretical_loss": 3.450401299843457, + "tokens_seen": 1866137600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044197979797979797, + "loss": 1.392, + "theoretical_loss": 3.450391058781001, + "tokens_seen": 1866203136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004419595959595959, + "loss": 1.3039, + "theoretical_loss": 3.450380818178872, + "tokens_seen": 1866268672 + }, + { + "epoch": 0.12, + "learning_rate": 0.000441939393939394, + "loss": 1.6445, + "theoretical_loss": 3.4503705780370337, + "tokens_seen": 1866334208 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.35972630977630615, + "objective/train/avg_token_score": 0.007249823771417141, + "objective/train/docs_used": 1055457, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1949776411056519, + "objective/train/original_loss": 4.569217681884766, + "objective/train/theoretical_loss": 3.4503603383554475, + "objective/train/tokens_used": 225258976, + "objective/train/unlikelihood_loss": 0.057365208864212036, + "theoretical_loss": 3.4503603383554475, + "tokens_seen": 1866399744 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044191919191919195, + "loss": 1.4593, + "theoretical_loss": 3.4503603383554475, + "tokens_seen": 1866399744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418989898989899, + "loss": 1.3262, + "theoretical_loss": 3.450350099134078, + "tokens_seen": 1866465280 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044187878787878786, + "loss": 1.3928, + "theoretical_loss": 3.450339860372888, + "tokens_seen": 1866530816 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044185858585858587, + "loss": 1.4387, + "theoretical_loss": 3.4503296220718402, + "tokens_seen": 1866596352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418383838383839, + "loss": 1.4918, + "theoretical_loss": 3.4503193842308986, + "tokens_seen": 1866661888 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044181818181818184, + "loss": 1.2656, + "theoretical_loss": 3.4503091468500253, + "tokens_seen": 1866727424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417979797979798, + "loss": 1.4871, + "theoretical_loss": 3.4502989099291845, + "tokens_seen": 1866792960 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044177777777777775, + "loss": 1.5025, + "theoretical_loss": 3.450288673468339, + "tokens_seen": 1866858496 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044175757575757576, + "loss": 1.3684, + "theoretical_loss": 3.4502784374674516, + "tokens_seen": 1866924032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417373737373738, + "loss": 1.2633, + "theoretical_loss": 3.4502682019264856, + "tokens_seen": 1866989568 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044171717171717173, + "loss": 1.432, + "theoretical_loss": 3.450257966845405, + "tokens_seen": 1867055104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416969696969697, + "loss": 1.385, + "theoretical_loss": 3.450247732224172, + "tokens_seen": 1867120640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416767676767677, + "loss": 1.3333, + "theoretical_loss": 3.4502374980627506, + "tokens_seen": 1867186176 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044165656565656566, + "loss": 1.5311, + "theoretical_loss": 3.4502272643611027, + "tokens_seen": 1867251712 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044163636363636367, + "loss": 1.2273, + "theoretical_loss": 3.4502170311191933, + "tokens_seen": 1867317248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416161616161616, + "loss": 1.3906, + "theoretical_loss": 3.4502067983369846, + "tokens_seen": 1867382784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415959595959596, + "loss": 1.4789, + "theoretical_loss": 3.45019656601444, + "tokens_seen": 1867448320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415757575757576, + "loss": 1.4171, + "theoretical_loss": 3.450186334151522, + "tokens_seen": 1867513856 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044155555555555555, + "loss": 1.6178, + "theoretical_loss": 3.4501761027481948, + "tokens_seen": 1867579392 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044153535353535356, + "loss": 1.3581, + "theoretical_loss": 3.4501658718044217, + "tokens_seen": 1867644928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415151515151515, + "loss": 1.2223, + "theoretical_loss": 3.4501556413201646, + "tokens_seen": 1867710464 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044149494949494953, + "loss": 1.1382, + "theoretical_loss": 3.450145411295389, + "tokens_seen": 1867776000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414747474747475, + "loss": 1.4542, + "theoretical_loss": 3.450135181730056, + "tokens_seen": 1867841536 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044145454545454544, + "loss": 1.5126, + "theoretical_loss": 3.4501249526241295, + "tokens_seen": 1867907072 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044143434343434345, + "loss": 1.5464, + "theoretical_loss": 3.450114723977573, + "tokens_seen": 1867972608 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.5038489699363708, + "objective/train/avg_token_score": 0.0115775465965271, + "objective/train/docs_used": 1056292, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7018463611602783, + "objective/train/original_loss": 4.302921295166016, + "objective/train/theoretical_loss": 3.4501044957903506, + "objective/train/tokens_used": 226897376, + "objective/train/unlikelihood_loss": 0.07719095051288605, + "theoretical_loss": 3.4501044957903506, + "tokens_seen": 1868038144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414141414141414, + "loss": 1.5629, + "theoretical_loss": 3.4501044957903506, + "tokens_seen": 1868038144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413939393939394, + "loss": 1.3905, + "theoretical_loss": 3.450094268062424, + "tokens_seen": 1868103680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413737373737374, + "loss": 1.514, + "theoretical_loss": 3.4500840407937567, + "tokens_seen": 1868169216 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044135353535353533, + "loss": 1.4503, + "theoretical_loss": 3.450073813984313, + "tokens_seen": 1868234752 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044133333333333335, + "loss": 1.3241, + "theoretical_loss": 3.4500635876340553, + "tokens_seen": 1868300288 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044131313131313136, + "loss": 1.3679, + "theoretical_loss": 3.450053361742947, + "tokens_seen": 1868365824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412929292929293, + "loss": 1.3124, + "theoretical_loss": 3.450043136310952, + "tokens_seen": 1868431360 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044127272727272727, + "loss": 1.4361, + "theoretical_loss": 3.450032911338032, + "tokens_seen": 1868496896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412525252525252, + "loss": 1.5088, + "theoretical_loss": 3.4500226868241524, + "tokens_seen": 1868562432 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044123232323232324, + "loss": 1.5715, + "theoretical_loss": 3.450012462769275, + "tokens_seen": 1868627968 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044121212121212125, + "loss": 1.3653, + "theoretical_loss": 3.450002239173364, + "tokens_seen": 1868693504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411919191919192, + "loss": 1.6154, + "theoretical_loss": 3.449992016036382, + "tokens_seen": 1868759040 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044117171717171716, + "loss": 1.5006, + "theoretical_loss": 3.449981793358292, + "tokens_seen": 1868824576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411515151515151, + "loss": 1.5315, + "theoretical_loss": 3.4499715711390584, + "tokens_seen": 1868890112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411313131313132, + "loss": 1.416, + "theoretical_loss": 3.4499613493786443, + "tokens_seen": 1868955648 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044111111111111114, + "loss": 1.7077, + "theoretical_loss": 3.4499511280770125, + "tokens_seen": 1869021184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410909090909091, + "loss": 1.4733, + "theoretical_loss": 3.4499409072341263, + "tokens_seen": 1869086720 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044107070707070706, + "loss": 1.2313, + "theoretical_loss": 3.4499306868499495, + "tokens_seen": 1869152256 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441050505050505, + "loss": 1.3513, + "theoretical_loss": 3.449920466924445, + "tokens_seen": 1869217792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410303030303031, + "loss": 1.4508, + "theoretical_loss": 3.4499102474575762, + "tokens_seen": 1869283328 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044101010101010103, + "loss": 1.4852, + "theoretical_loss": 3.4499000284493073, + "tokens_seen": 1869348864 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440989898989899, + "loss": 1.4368, + "theoretical_loss": 3.4498898098996, + "tokens_seen": 1869414400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044096969696969695, + "loss": 1.3618, + "theoretical_loss": 3.4498795918084193, + "tokens_seen": 1869479936 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044094949494949496, + "loss": 1.3093, + "theoretical_loss": 3.4498693741757274, + "tokens_seen": 1869545472 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044092929292929297, + "loss": 1.3623, + "theoretical_loss": 3.4498591570014883, + "tokens_seen": 1869611008 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.49095797538757324, + "objective/train/avg_token_score": 0.0304701067507267, + "objective/train/docs_used": 1057648, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6677335500717163, + "objective/train/original_loss": 4.48096227645874, + "objective/train/theoretical_loss": 3.4498489402856647, + "objective/train/tokens_used": 228535776, + "objective/train/unlikelihood_loss": 0.04762193188071251, + "theoretical_loss": 3.4498489402856647, + "tokens_seen": 1869676544 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044090909090909093, + "loss": 1.5815, + "theoretical_loss": 3.4498489402856647, + "tokens_seen": 1869676544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408888888888889, + "loss": 1.3001, + "theoretical_loss": 3.4498387240282207, + "tokens_seen": 1869742080 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044086868686868684, + "loss": 1.6185, + "theoretical_loss": 3.4498285082291194, + "tokens_seen": 1869807616 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044084848484848485, + "loss": 1.5549, + "theoretical_loss": 3.449818292888324, + "tokens_seen": 1869873152 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044082828282828286, + "loss": 1.3115, + "theoretical_loss": 3.4498080780057983, + "tokens_seen": 1869938688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408080808080808, + "loss": 1.3658, + "theoretical_loss": 3.449797863581505, + "tokens_seen": 1870004224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407878787878788, + "loss": 1.5948, + "theoretical_loss": 3.4497876496154083, + "tokens_seen": 1870069760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407676767676768, + "loss": 1.399, + "theoretical_loss": 3.4497774361074707, + "tokens_seen": 1870135296 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044074747474747474, + "loss": 1.3492, + "theoretical_loss": 3.4497672230576564, + "tokens_seen": 1870200832 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044072727272727276, + "loss": 1.3206, + "theoretical_loss": 3.4497570104659285, + "tokens_seen": 1870266368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407070707070707, + "loss": 1.6108, + "theoretical_loss": 3.4497467983322503, + "tokens_seen": 1870331904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044068686868686867, + "loss": 1.559, + "theoretical_loss": 3.4497365866565852, + "tokens_seen": 1870397440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406666666666667, + "loss": 1.3866, + "theoretical_loss": 3.4497263754388965, + "tokens_seen": 1870462976 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044064646464646464, + "loss": 1.2945, + "theoretical_loss": 3.449716164679148, + "tokens_seen": 1870528512 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044062626262626265, + "loss": 1.5109, + "theoretical_loss": 3.449705954377303, + "tokens_seen": 1870594048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406060606060606, + "loss": 1.4244, + "theoretical_loss": 3.449695744533325, + "tokens_seen": 1870659584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405858585858586, + "loss": 1.4769, + "theoretical_loss": 3.449685535147177, + "tokens_seen": 1870725120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405656565656566, + "loss": 1.5157, + "theoretical_loss": 3.4496753262188227, + "tokens_seen": 1870790656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044054545454545453, + "loss": 1.2994, + "theoretical_loss": 3.4496651177482254, + "tokens_seen": 1870856192 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044052525252525254, + "loss": 1.4714, + "theoretical_loss": 3.449654909735349, + "tokens_seen": 1870921728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405050505050505, + "loss": 1.3804, + "theoretical_loss": 3.449644702180156, + "tokens_seen": 1870987264 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404848484848485, + "loss": 1.4414, + "theoretical_loss": 3.4496344950826114, + "tokens_seen": 1871052800 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044046464646464647, + "loss": 1.3298, + "theoretical_loss": 3.4496242884426773, + "tokens_seen": 1871118336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404444444444445, + "loss": 1.3629, + "theoretical_loss": 3.449614082260317, + "tokens_seen": 1871183872 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044042424242424243, + "loss": 1.442, + "theoretical_loss": 3.4496038765354955, + "tokens_seen": 1871249408 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.4029814302921295, + "objective/train/avg_token_score": 0.022246306762099266, + "objective/train/docs_used": 1058498, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2657883167266846, + "objective/train/original_loss": 4.2054595947265625, + "objective/train/theoretical_loss": 3.449593671268175, + "objective/train/tokens_used": 230174176, + "objective/train/unlikelihood_loss": 0.0596332885324955, + "theoretical_loss": 3.449593671268175, + "tokens_seen": 1871314944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044040404040404044, + "loss": 1.2121, + "theoretical_loss": 3.449593671268175, + "tokens_seen": 1871314944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403838383838384, + "loss": 1.4529, + "theoretical_loss": 3.449583466458319, + "tokens_seen": 1871380480 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044036363636363636, + "loss": 1.3306, + "theoretical_loss": 3.4495732621058917, + "tokens_seen": 1871446016 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044034343434343437, + "loss": 1.7467, + "theoretical_loss": 3.449563058210856, + "tokens_seen": 1871511552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403232323232323, + "loss": 1.2752, + "theoretical_loss": 3.4495528547731755, + "tokens_seen": 1871577088 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044030303030303034, + "loss": 1.337, + "theoretical_loss": 3.4495426517928136, + "tokens_seen": 1871642624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402828282828283, + "loss": 1.2836, + "theoretical_loss": 3.449532449269734, + "tokens_seen": 1871708160 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044026262626262625, + "loss": 1.3727, + "theoretical_loss": 3.4495222472039, + "tokens_seen": 1871773696 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044024242424242426, + "loss": 1.3912, + "theoretical_loss": 3.449512045595275, + "tokens_seen": 1871839232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402222222222222, + "loss": 1.4787, + "theoretical_loss": 3.449501844443823, + "tokens_seen": 1871904768 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044020202020202023, + "loss": 1.5051, + "theoretical_loss": 3.449491643749507, + "tokens_seen": 1871970304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401818181818182, + "loss": 1.6093, + "theoretical_loss": 3.449481443512291, + "tokens_seen": 1872035840 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044016161616161614, + "loss": 1.4234, + "theoretical_loss": 3.449471243732138, + "tokens_seen": 1872101376 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044014141414141416, + "loss": 1.6637, + "theoretical_loss": 3.449461044409012, + "tokens_seen": 1872166912 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044012121212121217, + "loss": 1.4772, + "theoretical_loss": 3.449450845542876, + "tokens_seen": 1872232448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401010101010101, + "loss": 1.491, + "theoretical_loss": 3.4494406471336934, + "tokens_seen": 1872297984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400808080808081, + "loss": 1.6623, + "theoretical_loss": 3.4494304491814285, + "tokens_seen": 1872363520 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044006060606060604, + "loss": 1.5462, + "theoretical_loss": 3.449420251686045, + "tokens_seen": 1872429056 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044004040404040405, + "loss": 1.4287, + "theoretical_loss": 3.4494100546475055, + "tokens_seen": 1872494592 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044002020202020206, + "loss": 1.532, + "theoretical_loss": 3.4493998580657736, + "tokens_seen": 1872560128 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044, + "loss": 1.2202, + "theoretical_loss": 3.4493896619408138, + "tokens_seen": 1872625664 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043997979797979797, + "loss": 1.4907, + "theoretical_loss": 3.449379466272589, + "tokens_seen": 1872691200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043995959595959593, + "loss": 1.61, + "theoretical_loss": 3.4493692710610624, + "tokens_seen": 1872756736 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439939393939394, + "loss": 1.4321, + "theoretical_loss": 3.449359076306198, + "tokens_seen": 1872822272 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043991919191919195, + "loss": 1.608, + "theoretical_loss": 3.4493488820079596, + "tokens_seen": 1872887808 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.39846041798591614, + "objective/train/avg_token_score": 0.02917778491973877, + "objective/train/docs_used": 1059136, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3530430793762207, + "objective/train/original_loss": 4.573091506958008, + "objective/train/theoretical_loss": 3.4493386881663106, + "objective/train/tokens_used": 231812576, + "objective/train/unlikelihood_loss": 0.08011636883020401, + "theoretical_loss": 3.4493386881663106, + "tokens_seen": 1872953344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398989898989899, + "loss": 1.4504, + "theoretical_loss": 3.4493386881663106, + "tokens_seen": 1872953344 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043987878787878787, + "loss": 1.2369, + "theoretical_loss": 3.4493284947812146, + "tokens_seen": 1873018880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398585858585858, + "loss": 1.5826, + "theoretical_loss": 3.4493183018526348, + "tokens_seen": 1873084416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398383838383839, + "loss": 1.2305, + "theoretical_loss": 3.449308109380535, + "tokens_seen": 1873149952 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043981818181818184, + "loss": 1.5691, + "theoretical_loss": 3.449297917364879, + "tokens_seen": 1873215488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397979797979798, + "loss": 1.5117, + "theoretical_loss": 3.44928772580563, + "tokens_seen": 1873281024 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043977777777777776, + "loss": 1.4064, + "theoretical_loss": 3.449277534702752, + "tokens_seen": 1873346560 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043975757575757577, + "loss": 1.6067, + "theoretical_loss": 3.4492673440562083, + "tokens_seen": 1873412096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397373737373738, + "loss": 1.4002, + "theoretical_loss": 3.449257153865963, + "tokens_seen": 1873477632 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043971717171717174, + "loss": 1.6485, + "theoretical_loss": 3.449246964131979, + "tokens_seen": 1873543168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396969696969697, + "loss": 1.4152, + "theoretical_loss": 3.4492367748542203, + "tokens_seen": 1873608704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043967676767676765, + "loss": 1.572, + "theoretical_loss": 3.4492265860326508, + "tokens_seen": 1873674240 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043965656565656566, + "loss": 1.545, + "theoretical_loss": 3.449216397667233, + "tokens_seen": 1873739776 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043963636363636367, + "loss": 1.3972, + "theoretical_loss": 3.449206209757932, + "tokens_seen": 1873805312 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043961616161616163, + "loss": 1.3669, + "theoretical_loss": 3.4491960223047107, + "tokens_seen": 1873870848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395959595959596, + "loss": 1.3002, + "theoretical_loss": 3.4491858353075324, + "tokens_seen": 1873936384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395757575757576, + "loss": 1.4357, + "theoretical_loss": 3.449175648766362, + "tokens_seen": 1874001920 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043955555555555555, + "loss": 1.432, + "theoretical_loss": 3.449165462681161, + "tokens_seen": 1874067456 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043953535353535357, + "loss": 1.3794, + "theoretical_loss": 3.4491552770518954, + "tokens_seen": 1874132992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395151515151515, + "loss": 1.5028, + "theoretical_loss": 3.449145091878527, + "tokens_seen": 1874198528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394949494949495, + "loss": 1.5145, + "theoretical_loss": 3.4491349071610204, + "tokens_seen": 1874264064 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394747474747475, + "loss": 1.3535, + "theoretical_loss": 3.449124722899339, + "tokens_seen": 1874329600 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043945454545454545, + "loss": 1.4699, + "theoretical_loss": 3.449114539093447, + "tokens_seen": 1874395136 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043943434343434346, + "loss": 1.3388, + "theoretical_loss": 3.449104355743307, + "tokens_seen": 1874460672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394141414141414, + "loss": 1.2571, + "theoretical_loss": 3.4490941728488838, + "tokens_seen": 1874526208 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.3365102708339691, + "objective/train/avg_token_score": 0.021979324519634247, + "objective/train/docs_used": 1059915, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1313660144805908, + "objective/train/original_loss": 4.815589904785156, + "objective/train/theoretical_loss": 3.44908399041014, + "objective/train/tokens_used": 233450976, + "objective/train/unlikelihood_loss": 0.04281238839030266, + "theoretical_loss": 3.44908399041014, + "tokens_seen": 1874591744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393939393939394, + "loss": 1.4928, + "theoretical_loss": 3.44908399041014, + "tokens_seen": 1874591744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393737373737374, + "loss": 1.3874, + "theoretical_loss": 3.4490738084270403, + "tokens_seen": 1874657280 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043935353535353534, + "loss": 1.4704, + "theoretical_loss": 3.4490636268995476, + "tokens_seen": 1874722816 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043933333333333335, + "loss": 1.5677, + "theoretical_loss": 3.4490534458276256, + "tokens_seen": 1874788352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393131313131313, + "loss": 1.499, + "theoretical_loss": 3.449043265211239, + "tokens_seen": 1874853888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392929292929293, + "loss": 1.5894, + "theoretical_loss": 3.4490330850503503, + "tokens_seen": 1874919424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392727272727273, + "loss": 1.4306, + "theoretical_loss": 3.4490229053449237, + "tokens_seen": 1874984960 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043925252525252523, + "loss": 1.3304, + "theoretical_loss": 3.449012726094923, + "tokens_seen": 1875050496 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043923232323232324, + "loss": 1.5022, + "theoretical_loss": 3.449002547300312, + "tokens_seen": 1875116032 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043921212121212125, + "loss": 1.3642, + "theoretical_loss": 3.4489923689610538, + "tokens_seen": 1875181568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391919191919192, + "loss": 1.3564, + "theoretical_loss": 3.4489821910771123, + "tokens_seen": 1875247104 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043917171717171717, + "loss": 1.4577, + "theoretical_loss": 3.448972013648452, + "tokens_seen": 1875312640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391515151515151, + "loss": 1.3256, + "theoretical_loss": 3.4489618366750356, + "tokens_seen": 1875378176 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043913131313131314, + "loss": 1.3953, + "theoretical_loss": 3.448951660156828, + "tokens_seen": 1875443712 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043911111111111115, + "loss": 1.44, + "theoretical_loss": 3.4489414840937913, + "tokens_seen": 1875509248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390909090909091, + "loss": 1.3612, + "theoretical_loss": 3.448931308485891, + "tokens_seen": 1875574784 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043907070707070706, + "loss": 1.6487, + "theoretical_loss": 3.4489211333330894, + "tokens_seen": 1875640320 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439050505050505, + "loss": 1.4251, + "theoretical_loss": 3.4489109586353512, + "tokens_seen": 1875705856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390303030303031, + "loss": 1.2871, + "theoretical_loss": 3.44890078439264, + "tokens_seen": 1875771392 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043901010101010104, + "loss": 1.5798, + "theoretical_loss": 3.448890610604918, + "tokens_seen": 1875836928 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438989898989899, + "loss": 1.3093, + "theoretical_loss": 3.4488804372721518, + "tokens_seen": 1875902464 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043896969696969695, + "loss": 1.6002, + "theoretical_loss": 3.448870264394303, + "tokens_seen": 1875968000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389494949494949, + "loss": 1.5905, + "theoretical_loss": 3.448860091971336, + "tokens_seen": 1876033536 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438929292929293, + "loss": 1.3809, + "theoretical_loss": 3.448849920003215, + "tokens_seen": 1876099072 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043890909090909093, + "loss": 1.5747, + "theoretical_loss": 3.4488397484899034, + "tokens_seen": 1876164608 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.3816593289375305, + "objective/train/avg_token_score": 0.014148054644465446, + "objective/train/docs_used": 1060624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0186139345169067, + "objective/train/original_loss": 3.7407007217407227, + "objective/train/theoretical_loss": 3.448829577431365, + "objective/train/tokens_used": 235089376, + "objective/train/unlikelihood_loss": 0.09927164018154144, + "theoretical_loss": 3.448829577431365, + "tokens_seen": 1876230144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388888888888889, + "loss": 1.2479, + "theoretical_loss": 3.448829577431365, + "tokens_seen": 1876230144 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043886868686868685, + "loss": 1.4699, + "theoretical_loss": 3.448819406827563, + "tokens_seen": 1876295680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388484848484848, + "loss": 1.3708, + "theoretical_loss": 3.448809236678462, + "tokens_seen": 1876361216 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043882828282828287, + "loss": 1.4538, + "theoretical_loss": 3.4487990669840256, + "tokens_seen": 1876426752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388080808080808, + "loss": 1.7556, + "theoretical_loss": 3.448788897744217, + "tokens_seen": 1876492288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387878787878788, + "loss": 1.3391, + "theoretical_loss": 3.4487787289590015, + "tokens_seen": 1876557824 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043876767676767674, + "loss": 1.5276, + "theoretical_loss": 3.4487685606283414, + "tokens_seen": 1876623360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387474747474748, + "loss": 1.6076, + "theoretical_loss": 3.4487583927522008, + "tokens_seen": 1876688896 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043872727272727276, + "loss": 1.3362, + "theoretical_loss": 3.4487482253305437, + "tokens_seen": 1876754432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387070707070707, + "loss": 1.4142, + "theoretical_loss": 3.4487380583633342, + "tokens_seen": 1876819968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386868686868687, + "loss": 1.6026, + "theoretical_loss": 3.4487278918505364, + "tokens_seen": 1876885504 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043866666666666663, + "loss": 1.4723, + "theoretical_loss": 3.448717725792113, + "tokens_seen": 1876951040 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386464646464647, + "loss": 1.6469, + "theoretical_loss": 3.448707560188028, + "tokens_seen": 1877016576 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043862626262626265, + "loss": 1.5528, + "theoretical_loss": 3.4486973950382467, + "tokens_seen": 1877082112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386060606060606, + "loss": 1.4923, + "theoretical_loss": 3.448687230342731, + "tokens_seen": 1877147648 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043858585858585857, + "loss": 1.1962, + "theoretical_loss": 3.448677066101446, + "tokens_seen": 1877213184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385656565656566, + "loss": 1.3383, + "theoretical_loss": 3.448666902314355, + "tokens_seen": 1877278720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385454545454546, + "loss": 1.354, + "theoretical_loss": 3.4486567389814224, + "tokens_seen": 1877344256 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043852525252525255, + "loss": 1.4967, + "theoretical_loss": 3.4486465761026115, + "tokens_seen": 1877409792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385050505050505, + "loss": 1.362, + "theoretical_loss": 3.448636413677886, + "tokens_seen": 1877475328 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043848484848484846, + "loss": 1.5587, + "theoretical_loss": 3.4486262517072106, + "tokens_seen": 1877540864 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043846464646464647, + "loss": 1.4487, + "theoretical_loss": 3.4486160901905487, + "tokens_seen": 1877606400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384444444444445, + "loss": 1.242, + "theoretical_loss": 3.448605929127864, + "tokens_seen": 1877671936 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043842424242424244, + "loss": 1.5602, + "theoretical_loss": 3.4485957685191204, + "tokens_seen": 1877737472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384040404040404, + "loss": 1.3082, + "theoretical_loss": 3.448585608364282, + "tokens_seen": 1877803008 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.49346286058425903, + "objective/train/avg_token_score": 0.00607198616489768, + "objective/train/docs_used": 1062058, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6114000082015991, + "objective/train/original_loss": 3.4861137866973877, + "objective/train/theoretical_loss": 3.4485754486633127, + "objective/train/tokens_used": 236727776, + "objective/train/unlikelihood_loss": 0.09068997949361801, + "theoretical_loss": 3.4485754486633127, + "tokens_seen": 1877868544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383838383838384, + "loss": 1.4889, + "theoretical_loss": 3.4485754486633127, + "tokens_seen": 1877868544 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043836363636363636, + "loss": 1.4036, + "theoretical_loss": 3.448565289416176, + "tokens_seen": 1877934080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383434343434344, + "loss": 1.5098, + "theoretical_loss": 3.448555130622836, + "tokens_seen": 1877999616 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043832323232323233, + "loss": 1.429, + "theoretical_loss": 3.4485449722832566, + "tokens_seen": 1878065152 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383030303030303, + "loss": 1.2384, + "theoretical_loss": 3.4485348143974024, + "tokens_seen": 1878130688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382828282828283, + "loss": 1.3225, + "theoretical_loss": 3.448524656965236, + "tokens_seen": 1878196224 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043826262626262626, + "loss": 1.5303, + "theoretical_loss": 3.448514499986722, + "tokens_seen": 1878261760 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043824242424242427, + "loss": 1.4846, + "theoretical_loss": 3.448504343461824, + "tokens_seen": 1878327296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382222222222222, + "loss": 1.4366, + "theoretical_loss": 3.448494187390507, + "tokens_seen": 1878392832 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043820202020202024, + "loss": 1.2521, + "theoretical_loss": 3.448484031772734, + "tokens_seen": 1878458368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381818181818182, + "loss": 1.2788, + "theoretical_loss": 3.4484738766084684, + "tokens_seen": 1878523904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043816161616161615, + "loss": 1.4862, + "theoretical_loss": 3.4484637218976752, + "tokens_seen": 1878589440 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043814141414141416, + "loss": 1.5265, + "theoretical_loss": 3.4484535676403176, + "tokens_seen": 1878654976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381212121212121, + "loss": 1.734, + "theoretical_loss": 3.44844341383636, + "tokens_seen": 1878720512 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043810101010101013, + "loss": 1.465, + "theoretical_loss": 3.448433260485766, + "tokens_seen": 1878786048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380808080808081, + "loss": 1.44, + "theoretical_loss": 3.4484231075885, + "tokens_seen": 1878851584 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043806060606060604, + "loss": 1.6416, + "theoretical_loss": 3.4484129551445255, + "tokens_seen": 1878917120 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043804040404040405, + "loss": 1.3597, + "theoretical_loss": 3.4484028031538063, + "tokens_seen": 1878982656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043802020202020206, + "loss": 1.7944, + "theoretical_loss": 3.4483926516163073, + "tokens_seen": 1879048192 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438, + "loss": 1.4988, + "theoretical_loss": 3.4483825005319915, + "tokens_seen": 1879113728 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437979797979798, + "loss": 1.2804, + "theoretical_loss": 3.448372349900823, + "tokens_seen": 1879179264 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043795959595959593, + "loss": 1.3335, + "theoretical_loss": 3.448362199722766, + "tokens_seen": 1879244800 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043793939393939395, + "loss": 1.5275, + "theoretical_loss": 3.4483520499977844, + "tokens_seen": 1879310336 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043791919191919196, + "loss": 1.4088, + "theoretical_loss": 3.4483419007258425, + "tokens_seen": 1879375872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004378989898989899, + "loss": 1.5582, + "theoretical_loss": 3.4483317519069043, + "tokens_seen": 1879441408 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.5173509120941162, + "objective/train/avg_token_score": 0.04670140892267227, + "objective/train/docs_used": 1062685, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7349787950515747, + "objective/train/original_loss": 3.844883918762207, + "objective/train/theoretical_loss": 3.4483216035409328, + "objective/train/tokens_used": 238366176, + "objective/train/unlikelihood_loss": 0.05952979996800423, + "theoretical_loss": 3.4483216035409328, + "tokens_seen": 1879506944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043787878787878787, + "loss": 1.3996, + "theoretical_loss": 3.4483216035409328, + "tokens_seen": 1879506944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043785858585858583, + "loss": 1.4732, + "theoretical_loss": 3.4483114556278927, + "tokens_seen": 1879572480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004378383838383839, + "loss": 1.5456, + "theoretical_loss": 3.448301308167748, + "tokens_seen": 1879638016 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043781818181818185, + "loss": 1.732, + "theoretical_loss": 3.448291161160463, + "tokens_seen": 1879703552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377979797979798, + "loss": 1.5276, + "theoretical_loss": 3.4482810146060014, + "tokens_seen": 1879769088 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043777777777777776, + "loss": 1.6654, + "theoretical_loss": 3.4482708685043266, + "tokens_seen": 1879834624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377575757575757, + "loss": 1.4529, + "theoretical_loss": 3.4482607228554034, + "tokens_seen": 1879900160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377373737373738, + "loss": 1.367, + "theoretical_loss": 3.448250577659196, + "tokens_seen": 1879965696 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043771717171717174, + "loss": 1.3748, + "theoretical_loss": 3.448240432915667, + "tokens_seen": 1880031232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376969696969697, + "loss": 1.3824, + "theoretical_loss": 3.4482302886247824, + "tokens_seen": 1880096768 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043767676767676766, + "loss": 1.4715, + "theoretical_loss": 3.448220144786505, + "tokens_seen": 1880162304 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043765656565656567, + "loss": 1.3517, + "theoretical_loss": 3.448210001400799, + "tokens_seen": 1880227840 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376363636363637, + "loss": 1.3135, + "theoretical_loss": 3.4481998584676288, + "tokens_seen": 1880293376 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043761616161616164, + "loss": 1.5466, + "theoretical_loss": 3.448189715986958, + "tokens_seen": 1880358912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375959595959596, + "loss": 1.5333, + "theoretical_loss": 3.4481795739587504, + "tokens_seen": 1880424448 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043757575757575755, + "loss": 1.3885, + "theoretical_loss": 3.448169432382971, + "tokens_seen": 1880489984 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043755555555555556, + "loss": 1.6391, + "theoretical_loss": 3.448159291259583, + "tokens_seen": 1880555520 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043753535353535357, + "loss": 1.7587, + "theoretical_loss": 3.448149150588551, + "tokens_seen": 1880621056 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043751515151515153, + "loss": 1.4384, + "theoretical_loss": 3.4481390103698386, + "tokens_seen": 1880686592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374949494949495, + "loss": 1.3425, + "theoretical_loss": 3.44812887060341, + "tokens_seen": 1880752128 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374747474747475, + "loss": 1.597, + "theoretical_loss": 3.448118731289229, + "tokens_seen": 1880817664 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043745454545454545, + "loss": 1.2963, + "theoretical_loss": 3.448108592427261, + "tokens_seen": 1880883200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043743434343434346, + "loss": 1.4757, + "theoretical_loss": 3.448098454017469, + "tokens_seen": 1880948736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374141414141414, + "loss": 1.3524, + "theoretical_loss": 3.4480883160598164, + "tokens_seen": 1881014272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373939393939394, + "loss": 1.672, + "theoretical_loss": 3.448078178554268, + "tokens_seen": 1881079808 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.41361191868782043, + "objective/train/avg_token_score": 0.008350033313035965, + "objective/train/docs_used": 1064316, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4659239053726196, + "objective/train/original_loss": 4.062493801116943, + "objective/train/theoretical_loss": 3.4480680415007887, + "objective/train/tokens_used": 240004576, + "objective/train/unlikelihood_loss": 0.06435056775808334, + "theoretical_loss": 3.4480680415007887, + "tokens_seen": 1881145344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373737373737374, + "loss": 1.3753, + "theoretical_loss": 3.4480680415007887, + "tokens_seen": 1881145344 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043735353535353535, + "loss": 1.301, + "theoretical_loss": 3.4480579048993416, + "tokens_seen": 1881210880 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043733333333333336, + "loss": 1.3698, + "theoretical_loss": 3.448047768749891, + "tokens_seen": 1881276416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373131313131313, + "loss": 1.6219, + "theoretical_loss": 3.448037633052401, + "tokens_seen": 1881341952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372929292929293, + "loss": 1.4685, + "theoretical_loss": 3.4480274978068355, + "tokens_seen": 1881407488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372727272727273, + "loss": 1.3703, + "theoretical_loss": 3.448017363013159, + "tokens_seen": 1881473024 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043725252525252524, + "loss": 1.3487, + "theoretical_loss": 3.4480072286713357, + "tokens_seen": 1881538560 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043723232323232325, + "loss": 1.1748, + "theoretical_loss": 3.4479970947813294, + "tokens_seen": 1881604096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372121212121212, + "loss": 1.4412, + "theoretical_loss": 3.447986961343104, + "tokens_seen": 1881669632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371919191919192, + "loss": 1.5357, + "theoretical_loss": 3.4479768283566243, + "tokens_seen": 1881735168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371717171717172, + "loss": 1.6326, + "theoretical_loss": 3.447966695821854, + "tokens_seen": 1881800704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043715151515151513, + "loss": 1.5148, + "theoretical_loss": 3.447956563738757, + "tokens_seen": 1881866240 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043713131313131314, + "loss": 1.6197, + "theoretical_loss": 3.4479464321072983, + "tokens_seen": 1881931776 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043711111111111115, + "loss": 1.3488, + "theoretical_loss": 3.4479363009274406, + "tokens_seen": 1881997312 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370909090909091, + "loss": 1.5204, + "theoretical_loss": 3.4479261701991497, + "tokens_seen": 1882062848 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043707070707070707, + "loss": 1.4517, + "theoretical_loss": 3.4479160399223887, + "tokens_seen": 1882128384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370505050505051, + "loss": 1.5259, + "theoretical_loss": 3.4479059100971217, + "tokens_seen": 1882193920 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043703030303030303, + "loss": 1.4554, + "theoretical_loss": 3.4478957807233135, + "tokens_seen": 1882259456 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043701010101010105, + "loss": 1.4525, + "theoretical_loss": 3.447885651800928, + "tokens_seen": 1882324992 + }, + { + "epoch": 0.13, + "learning_rate": 0.000436989898989899, + "loss": 1.2467, + "theoretical_loss": 3.4478755233299294, + "tokens_seen": 1882390528 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043696969696969696, + "loss": 1.3973, + "theoretical_loss": 3.4478653953102816, + "tokens_seen": 1882456064 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043694949494949497, + "loss": 1.3898, + "theoretical_loss": 3.4478552677419487, + "tokens_seen": 1882521600 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043692929292929293, + "loss": 1.2577, + "theoretical_loss": 3.4478451406248953, + "tokens_seen": 1882587136 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043690909090909094, + "loss": 1.4622, + "theoretical_loss": 3.4478350139590854, + "tokens_seen": 1882652672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004368888888888889, + "loss": 1.5288, + "theoretical_loss": 3.447824887744483, + "tokens_seen": 1882718208 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.41856059432029724, + "objective/train/avg_token_score": 0.06959132105112076, + "objective/train/docs_used": 1064917, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4428768157958984, + "objective/train/original_loss": 4.215625286102295, + "objective/train/theoretical_loss": 3.447814761981053, + "objective/train/tokens_used": 241642976, + "objective/train/unlikelihood_loss": 0.06671393662691116, + "theoretical_loss": 3.447814761981053, + "tokens_seen": 1882783744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043686868686868685, + "loss": 1.6306, + "theoretical_loss": 3.447814761981053, + "tokens_seen": 1882783744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043684848484848486, + "loss": 1.304, + "theoretical_loss": 3.447804636668759, + "tokens_seen": 1882849280 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004368282828282829, + "loss": 1.347, + "theoretical_loss": 3.4477945118075652, + "tokens_seen": 1882914816 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043680808080808083, + "loss": 1.4824, + "theoretical_loss": 3.4477843873974354, + "tokens_seen": 1882980352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367878787878788, + "loss": 1.3546, + "theoretical_loss": 3.447774263438335, + "tokens_seen": 1883045888 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043676767676767674, + "loss": 1.3092, + "theoretical_loss": 3.447764139930227, + "tokens_seen": 1883111424 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043674747474747476, + "loss": 1.3361, + "theoretical_loss": 3.4477540168730765, + "tokens_seen": 1883176960 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043672727272727277, + "loss": 1.2515, + "theoretical_loss": 3.4477438942668472, + "tokens_seen": 1883242496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367070707070707, + "loss": 1.3784, + "theoretical_loss": 3.447733772111503, + "tokens_seen": 1883308032 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366868686868687, + "loss": 1.4131, + "theoretical_loss": 3.447723650407009, + "tokens_seen": 1883373568 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043666666666666664, + "loss": 1.3234, + "theoretical_loss": 3.447713529153329, + "tokens_seen": 1883439104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366464646464647, + "loss": 1.3278, + "theoretical_loss": 3.447703408350427, + "tokens_seen": 1883504640 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043662626262626266, + "loss": 1.3996, + "theoretical_loss": 3.447693287998268, + "tokens_seen": 1883570176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366060606060606, + "loss": 1.3433, + "theoretical_loss": 3.447683168096815, + "tokens_seen": 1883635712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365858585858586, + "loss": 1.5963, + "theoretical_loss": 3.447673048646034, + "tokens_seen": 1883701248 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043656565656565653, + "loss": 1.5976, + "theoretical_loss": 3.4476629296458876, + "tokens_seen": 1883766784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365454545454546, + "loss": 1.4234, + "theoretical_loss": 3.447652811096341, + "tokens_seen": 1883832320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043652525252525255, + "loss": 1.4693, + "theoretical_loss": 3.447642692997358, + "tokens_seen": 1883897856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365050505050505, + "loss": 1.3536, + "theoretical_loss": 3.4476325753489028, + "tokens_seen": 1883963392 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043648484848484847, + "loss": 1.5021, + "theoretical_loss": 3.4476224581509403, + "tokens_seen": 1884028928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364646464646465, + "loss": 1.4861, + "theoretical_loss": 3.447612341403434, + "tokens_seen": 1884094464 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364444444444445, + "loss": 1.4886, + "theoretical_loss": 3.4476022251063485, + "tokens_seen": 1884160000 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043642424242424244, + "loss": 1.007, + "theoretical_loss": 3.447592109259648, + "tokens_seen": 1884225536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364040404040404, + "loss": 1.3627, + "theoretical_loss": 3.447581993863297, + "tokens_seen": 1884291072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043638383838383836, + "loss": 1.3705, + "theoretical_loss": 3.44757187891726, + "tokens_seen": 1884356608 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.40738025307655334, + "objective/train/avg_token_score": 0.015582275576889515, + "objective/train/docs_used": 1065907, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.418578028678894, + "objective/train/original_loss": 4.257914066314697, + "objective/train/theoretical_loss": 3.4475617644215006, + "objective/train/tokens_used": 243281376, + "objective/train/unlikelihood_loss": 0.0637265294790268, + "theoretical_loss": 3.4475617644215006, + "tokens_seen": 1884422144 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043636363636363637, + "loss": 1.5404, + "theoretical_loss": 3.4475617644215006, + "tokens_seen": 1884422144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363434343434344, + "loss": 1.2178, + "theoretical_loss": 3.447551650375984, + "tokens_seen": 1884487680 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043632323232323234, + "loss": 1.4998, + "theoretical_loss": 3.4475415367806734, + "tokens_seen": 1884553216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363030303030303, + "loss": 1.3251, + "theoretical_loss": 3.4475314236355343, + "tokens_seen": 1884618752 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362828282828283, + "loss": 1.4469, + "theoretical_loss": 3.44752131094053, + "tokens_seen": 1884684288 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043626262626262626, + "loss": 1.5319, + "theoretical_loss": 3.447511198695625, + "tokens_seen": 1884749824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362424242424243, + "loss": 1.3746, + "theoretical_loss": 3.4475010869007843, + "tokens_seen": 1884815360 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043622222222222223, + "loss": 1.2662, + "theoretical_loss": 3.4474909755559713, + "tokens_seen": 1884880896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362020202020202, + "loss": 1.4889, + "theoretical_loss": 3.4474808646611512, + "tokens_seen": 1884946432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361818181818182, + "loss": 1.2985, + "theoretical_loss": 3.4474707542162877, + "tokens_seen": 1885011968 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043616161616161616, + "loss": 1.4024, + "theoretical_loss": 3.447460644221345, + "tokens_seen": 1885077504 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043614141414141417, + "loss": 1.6368, + "theoretical_loss": 3.447450534676288, + "tokens_seen": 1885143040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361212121212121, + "loss": 1.4649, + "theoretical_loss": 3.447440425581081, + "tokens_seen": 1885208576 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043610101010101013, + "loss": 1.3478, + "theoretical_loss": 3.447430316935688, + "tokens_seen": 1885274112 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360808080808081, + "loss": 1.1866, + "theoretical_loss": 3.4474202087400734, + "tokens_seen": 1885339648 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043606060606060605, + "loss": 1.4361, + "theoretical_loss": 3.447410100994202, + "tokens_seen": 1885405184 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043604040404040406, + "loss": 1.5223, + "theoretical_loss": 3.4473999936980375, + "tokens_seen": 1885470720 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436020202020202, + "loss": 1.549, + "theoretical_loss": 3.4473898868515445, + "tokens_seen": 1885536256 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436, + "loss": 1.2515, + "theoretical_loss": 3.447379780454688, + "tokens_seen": 1885601792 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435979797979798, + "loss": 1.5817, + "theoretical_loss": 3.4473696745074314, + "tokens_seen": 1885667328 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043595959595959594, + "loss": 1.5611, + "theoretical_loss": 3.4473595690097394, + "tokens_seen": 1885732864 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043593939393939395, + "loss": 1.6062, + "theoretical_loss": 3.4473494639615767, + "tokens_seen": 1885798400 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043591919191919196, + "loss": 1.3081, + "theoretical_loss": 3.447339359362907, + "tokens_seen": 1885863936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358989898989899, + "loss": 1.5584, + "theoretical_loss": 3.447329255213696, + "tokens_seen": 1885929472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358787878787879, + "loss": 1.362, + "theoretical_loss": 3.4473191515139066, + "tokens_seen": 1885995008 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.4603494703769684, + "objective/train/avg_token_score": 0.004963160492479801, + "objective/train/docs_used": 1066425, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4913145303726196, + "objective/train/original_loss": 4.012363910675049, + "objective/train/theoretical_loss": 3.447309048263504, + "objective/train/tokens_used": 244919776, + "objective/train/unlikelihood_loss": 0.06790236383676529, + "theoretical_loss": 3.447309048263504, + "tokens_seen": 1886060544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043585858585858583, + "loss": 1.3731, + "theoretical_loss": 3.447309048263504, + "tokens_seen": 1886060544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043583838383838384, + "loss": 1.3272, + "theoretical_loss": 3.4472989454624523, + "tokens_seen": 1886126080 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043581818181818186, + "loss": 1.4449, + "theoretical_loss": 3.447288843110716, + "tokens_seen": 1886191616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357979797979798, + "loss": 1.4432, + "theoretical_loss": 3.4472787412082595, + "tokens_seen": 1886257152 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043577777777777777, + "loss": 1.5336, + "theoretical_loss": 3.4472686397550474, + "tokens_seen": 1886322688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357575757575757, + "loss": 1.4763, + "theoretical_loss": 3.4472585387510435, + "tokens_seen": 1886388224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357373737373738, + "loss": 1.3966, + "theoretical_loss": 3.447248438196213, + "tokens_seen": 1886453760 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043571717171717175, + "loss": 1.4625, + "theoretical_loss": 3.4472383380905205, + "tokens_seen": 1886519296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356969696969697, + "loss": 1.4471, + "theoretical_loss": 3.4472282384339294, + "tokens_seen": 1886584832 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043567676767676766, + "loss": 1.3535, + "theoretical_loss": 3.4472181392264045, + "tokens_seen": 1886650368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356565656565656, + "loss": 1.3392, + "theoretical_loss": 3.4472080404679106, + "tokens_seen": 1886715904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356363636363637, + "loss": 1.4189, + "theoretical_loss": 3.447197942158412, + "tokens_seen": 1886781440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043561616161616164, + "loss": 1.5119, + "theoretical_loss": 3.447187844297873, + "tokens_seen": 1886846976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355959595959596, + "loss": 1.6033, + "theoretical_loss": 3.447177746886258, + "tokens_seen": 1886912512 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043557575757575755, + "loss": 1.5703, + "theoretical_loss": 3.4471676499235318, + "tokens_seen": 1886978048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355555555555555, + "loss": 1.4541, + "theoretical_loss": 3.4471575534096583, + "tokens_seen": 1887043584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355353535353536, + "loss": 1.4322, + "theoretical_loss": 3.4471474573446024, + "tokens_seen": 1887109120 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043551515151515153, + "loss": 1.485, + "theoretical_loss": 3.4471373617283287, + "tokens_seen": 1887174656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354949494949495, + "loss": 1.4842, + "theoretical_loss": 3.447127266560801, + "tokens_seen": 1887240192 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043547474747474745, + "loss": 1.4015, + "theoretical_loss": 3.4471171718419846, + "tokens_seen": 1887305728 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043545454545454546, + "loss": 1.5789, + "theoretical_loss": 3.447107077571843, + "tokens_seen": 1887371264 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043543434343434347, + "loss": 1.5174, + "theoretical_loss": 3.447096983750342, + "tokens_seen": 1887436800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354141414141414, + "loss": 1.2054, + "theoretical_loss": 3.447086890377445, + "tokens_seen": 1887502336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353939393939394, + "loss": 1.3724, + "theoretical_loss": 3.447076797453117, + "tokens_seen": 1887567872 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043537373737373734, + "loss": 1.6524, + "theoretical_loss": 3.4470667049773214, + "tokens_seen": 1887633408 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.4772116243839264, + "objective/train/avg_token_score": 0.018777167424559593, + "objective/train/docs_used": 1067085, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5127924680709839, + "objective/train/original_loss": 4.050947666168213, + "objective/train/theoretical_loss": 3.4470566129500244, + "objective/train/tokens_used": 246558176, + "objective/train/unlikelihood_loss": 0.05088551715016365, + "theoretical_loss": 3.4470566129500244, + "tokens_seen": 1887698944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043535353535353535, + "loss": 1.3626, + "theoretical_loss": 3.4470566129500244, + "tokens_seen": 1887698944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043533333333333336, + "loss": 1.3871, + "theoretical_loss": 3.4470465213711896, + "tokens_seen": 1887764480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353131313131313, + "loss": 1.5453, + "theoretical_loss": 3.4470364302407814, + "tokens_seen": 1887830016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352929292929293, + "loss": 1.4407, + "theoretical_loss": 3.447026339558765, + "tokens_seen": 1887895552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352727272727273, + "loss": 1.4609, + "theoretical_loss": 3.4470162493251038, + "tokens_seen": 1887961088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352525252525253, + "loss": 1.4642, + "theoretical_loss": 3.4470061595397627, + "tokens_seen": 1888026624 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043523232323232325, + "loss": 1.2666, + "theoretical_loss": 3.446996070202707, + "tokens_seen": 1888092160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352121212121212, + "loss": 1.2846, + "theoretical_loss": 3.446985981313901, + "tokens_seen": 1888157696 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043519191919191917, + "loss": 1.4019, + "theoretical_loss": 3.4469758928733083, + "tokens_seen": 1888223232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351717171717172, + "loss": 1.4626, + "theoretical_loss": 3.446965804880894, + "tokens_seen": 1888288768 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351515151515152, + "loss": 1.1625, + "theoretical_loss": 3.446955717336623, + "tokens_seen": 1888354304 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043513131313131315, + "loss": 1.5753, + "theoretical_loss": 3.4469456302404597, + "tokens_seen": 1888419840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351111111111111, + "loss": 1.4806, + "theoretical_loss": 3.4469355435923683, + "tokens_seen": 1888485376 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350909090909091, + "loss": 1.5697, + "theoretical_loss": 3.4469254573923136, + "tokens_seen": 1888550912 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043507070707070707, + "loss": 1.4095, + "theoretical_loss": 3.4469153716402596, + "tokens_seen": 1888616448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350505050505051, + "loss": 1.5738, + "theoretical_loss": 3.4469052863361713, + "tokens_seen": 1888681984 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043503030303030304, + "loss": 1.4386, + "theoretical_loss": 3.4468952014800136, + "tokens_seen": 1888747520 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435010101010101, + "loss": 1.4282, + "theoretical_loss": 3.4468851170717505, + "tokens_seen": 1888813056 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434989898989899, + "loss": 1.4533, + "theoretical_loss": 3.446875033111347, + "tokens_seen": 1888878592 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043496969696969696, + "loss": 1.4092, + "theoretical_loss": 3.4468649495987678, + "tokens_seen": 1888944128 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434949494949495, + "loss": 1.4225, + "theoretical_loss": 3.4468548665339767, + "tokens_seen": 1889009664 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043492929292929293, + "loss": 1.1841, + "theoretical_loss": 3.446844783916939, + "tokens_seen": 1889075200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043490909090909094, + "loss": 1.4895, + "theoretical_loss": 3.4468347017476186, + "tokens_seen": 1889140736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348888888888889, + "loss": 1.4007, + "theoretical_loss": 3.446824620025981, + "tokens_seen": 1889206272 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043486868686868686, + "loss": 1.3433, + "theoretical_loss": 3.44681453875199, + "tokens_seen": 1889271808 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.4841764271259308, + "objective/train/avg_token_score": 0.00916412752121687, + "objective/train/docs_used": 1068281, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6944667100906372, + "objective/train/original_loss": 4.594786643981934, + "objective/train/theoretical_loss": 3.4468044579256105, + "objective/train/tokens_used": 248196576, + "objective/train/unlikelihood_loss": 0.028151439502835274, + "theoretical_loss": 3.4468044579256105, + "tokens_seen": 1889337344 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043484848484848487, + "loss": 1.437, + "theoretical_loss": 3.4468044579256105, + "tokens_seen": 1889337344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348282828282828, + "loss": 1.4151, + "theoretical_loss": 3.446794377546807, + "tokens_seen": 1889402880 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043480808080808084, + "loss": 1.3273, + "theoretical_loss": 3.4467842976155447, + "tokens_seen": 1889468416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347878787878788, + "loss": 1.5779, + "theoretical_loss": 3.4467742181317873, + "tokens_seen": 1889533952 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043476767676767675, + "loss": 1.5525, + "theoretical_loss": 3.4467641390955, + "tokens_seen": 1889599488 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043474747474747476, + "loss": 1.4035, + "theoretical_loss": 3.446754060506647, + "tokens_seen": 1889665024 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043472727272727277, + "loss": 1.4794, + "theoretical_loss": 3.4467439823651933, + "tokens_seen": 1889730560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043470707070707073, + "loss": 1.2579, + "theoretical_loss": 3.4467339046711034, + "tokens_seen": 1889796096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346868686868687, + "loss": 1.3739, + "theoretical_loss": 3.446723827424342, + "tokens_seen": 1889861632 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043466666666666664, + "loss": 1.4137, + "theoretical_loss": 3.4467137506248733, + "tokens_seen": 1889927168 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043464646464646465, + "loss": 1.4186, + "theoretical_loss": 3.446703674272663, + "tokens_seen": 1889992704 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043462626262626267, + "loss": 1.4476, + "theoretical_loss": 3.4466935983676743, + "tokens_seen": 1890058240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346060606060606, + "loss": 1.5298, + "theoretical_loss": 3.446683522909873, + "tokens_seen": 1890123776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345858585858586, + "loss": 1.4053, + "theoretical_loss": 3.446673447899223, + "tokens_seen": 1890189312 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043456565656565654, + "loss": 1.4032, + "theoretical_loss": 3.4466633733356895, + "tokens_seen": 1890254848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345454545454546, + "loss": 1.488, + "theoretical_loss": 3.4466532992192365, + "tokens_seen": 1890320384 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043452525252525256, + "loss": 1.3714, + "theoretical_loss": 3.4466432255498294, + "tokens_seen": 1890385920 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345050505050505, + "loss": 1.2779, + "theoretical_loss": 3.4466331523274327, + "tokens_seen": 1890451456 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043448484848484847, + "loss": 1.6398, + "theoretical_loss": 3.446623079552011, + "tokens_seen": 1890516992 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043446464646464643, + "loss": 1.319, + "theoretical_loss": 3.4466130072235286, + "tokens_seen": 1890582528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344444444444445, + "loss": 1.5699, + "theoretical_loss": 3.4466029353419505, + "tokens_seen": 1890648064 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043442424242424245, + "loss": 1.4971, + "theoretical_loss": 3.4465928639072416, + "tokens_seen": 1890713600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344040404040404, + "loss": 1.5592, + "theoretical_loss": 3.446582792919366, + "tokens_seen": 1890779136 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043438383838383836, + "loss": 1.4512, + "theoretical_loss": 3.446572722378289, + "tokens_seen": 1890844672 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343636363636364, + "loss": 1.365, + "theoretical_loss": 3.4465626522839745, + "tokens_seen": 1890910208 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.34188660979270935, + "objective/train/avg_token_score": 0.036958470940589905, + "objective/train/docs_used": 1069021, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.070196270942688, + "objective/train/original_loss": 4.393176555633545, + "objective/train/theoretical_loss": 3.4465525826363885, + "objective/train/tokens_used": 249834976, + "objective/train/unlikelihood_loss": 0.0641588643193245, + "theoretical_loss": 3.4465525826363885, + "tokens_seen": 1890975744 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343434343434344, + "loss": 1.2818, + "theoretical_loss": 3.4465525826363885, + "tokens_seen": 1890975744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043432323232323234, + "loss": 1.4453, + "theoretical_loss": 3.4465425134354946, + "tokens_seen": 1891041280 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343030303030303, + "loss": 1.2402, + "theoretical_loss": 3.4465324446812575, + "tokens_seen": 1891106816 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043428282828282826, + "loss": 1.1511, + "theoretical_loss": 3.4465223763736423, + "tokens_seen": 1891172352 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043426262626262627, + "loss": 1.3436, + "theoretical_loss": 3.446512308512614, + "tokens_seen": 1891237888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342424242424243, + "loss": 1.5911, + "theoretical_loss": 3.446502241098137, + "tokens_seen": 1891303424 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043422222222222224, + "loss": 1.6752, + "theoretical_loss": 3.4464921741301757, + "tokens_seen": 1891368960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342020202020202, + "loss": 1.6683, + "theoretical_loss": 3.4464821076086953, + "tokens_seen": 1891434496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341818181818182, + "loss": 1.3651, + "theoretical_loss": 3.44647204153366, + "tokens_seen": 1891500032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043416161616161616, + "loss": 1.2549, + "theoretical_loss": 3.4464619759050352, + "tokens_seen": 1891565568 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043414141414141417, + "loss": 1.3456, + "theoretical_loss": 3.4464519107227853, + "tokens_seen": 1891631104 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043412121212121213, + "loss": 1.3359, + "theoretical_loss": 3.4464418459868753, + "tokens_seen": 1891696640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341010101010101, + "loss": 1.3899, + "theoretical_loss": 3.446431781697269, + "tokens_seen": 1891762176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340808080808081, + "loss": 1.3479, + "theoretical_loss": 3.4464217178539327, + "tokens_seen": 1891827712 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043406060606060605, + "loss": 1.3248, + "theoretical_loss": 3.4464116544568295, + "tokens_seen": 1891893248 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043404040404040406, + "loss": 1.4866, + "theoretical_loss": 3.446401591505925, + "tokens_seen": 1891958784 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434020202020202, + "loss": 1.5089, + "theoretical_loss": 3.4463915290011844, + "tokens_seen": 1892024320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043400000000000003, + "loss": 1.3789, + "theoretical_loss": 3.446381466942572, + "tokens_seen": 1892089856 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433979797979798, + "loss": 1.4955, + "theoretical_loss": 3.4463714053300523, + "tokens_seen": 1892155392 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043395959595959595, + "loss": 1.3215, + "theoretical_loss": 3.44636134416359, + "tokens_seen": 1892220928 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043393939393939396, + "loss": 1.3518, + "theoretical_loss": 3.4463512834431507, + "tokens_seen": 1892286464 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339191919191919, + "loss": 1.3355, + "theoretical_loss": 3.446341223168698, + "tokens_seen": 1892352000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338989898989899, + "loss": 1.2708, + "theoretical_loss": 3.446331163340198, + "tokens_seen": 1892417536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338787878787879, + "loss": 1.2829, + "theoretical_loss": 3.446321103957615, + "tokens_seen": 1892483072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043385858585858584, + "loss": 1.3394, + "theoretical_loss": 3.4463110450209133, + "tokens_seen": 1892548608 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.34042033553123474, + "objective/train/avg_token_score": 0.015821443870663643, + "objective/train/docs_used": 1070317, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1134051084518433, + "objective/train/original_loss": 4.933949947357178, + "objective/train/theoretical_loss": 3.4463009865300585, + "objective/train/tokens_used": 251473376, + "objective/train/unlikelihood_loss": 0.06462860852479935, + "theoretical_loss": 3.4463009865300585, + "tokens_seen": 1892614144 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043383838383838385, + "loss": 1.2695, + "theoretical_loss": 3.4463009865300585, + "tokens_seen": 1892614144 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043381818181818186, + "loss": 1.4255, + "theoretical_loss": 3.4462909284850145, + "tokens_seen": 1892679680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337979797979798, + "loss": 1.2214, + "theoretical_loss": 3.4462808708857464, + "tokens_seen": 1892745216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337777777777778, + "loss": 1.2181, + "theoretical_loss": 3.4462708137322196, + "tokens_seen": 1892810752 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043375757575757573, + "loss": 1.5435, + "theoretical_loss": 3.446260757024398, + "tokens_seen": 1892876288 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043373737373737374, + "loss": 1.3984, + "theoretical_loss": 3.4462507007622474, + "tokens_seen": 1892941824 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043371717171717175, + "loss": 1.3259, + "theoretical_loss": 3.4462406449457315, + "tokens_seen": 1893007360 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336969696969697, + "loss": 1.3695, + "theoretical_loss": 3.4462305895748164, + "tokens_seen": 1893072896 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043367676767676767, + "loss": 1.2128, + "theoretical_loss": 3.4462205346494654, + "tokens_seen": 1893138432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336565656565656, + "loss": 1.6229, + "theoretical_loss": 3.446210480169645, + "tokens_seen": 1893203968 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043363636363636364, + "loss": 1.2829, + "theoretical_loss": 3.446200426135319, + "tokens_seen": 1893269504 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043361616161616165, + "loss": 1.6587, + "theoretical_loss": 3.4461903725464524, + "tokens_seen": 1893335040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335959595959596, + "loss": 1.1699, + "theoretical_loss": 3.4461803194030103, + "tokens_seen": 1893400576 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043357575757575756, + "loss": 1.2126, + "theoretical_loss": 3.446170266704957, + "tokens_seen": 1893466112 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043355555555555557, + "loss": 1.2627, + "theoretical_loss": 3.446160214452258, + "tokens_seen": 1893531648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335353535353536, + "loss": 1.3907, + "theoretical_loss": 3.4461501626448783, + "tokens_seen": 1893597184 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043351515151515154, + "loss": 1.4824, + "theoretical_loss": 3.446140111282782, + "tokens_seen": 1893662720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334949494949495, + "loss": 1.3936, + "theoretical_loss": 3.4461300603659337, + "tokens_seen": 1893728256 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043347474747474745, + "loss": 1.3941, + "theoretical_loss": 3.4461200098942997, + "tokens_seen": 1893793792 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043345454545454546, + "loss": 1.3449, + "theoretical_loss": 3.4461099598678437, + "tokens_seen": 1893859328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334343434343435, + "loss": 1.7087, + "theoretical_loss": 3.446099910286531, + "tokens_seen": 1893924864 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043341414141414143, + "loss": 1.8037, + "theoretical_loss": 3.446089861150326, + "tokens_seen": 1893990400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333939393939394, + "loss": 1.5121, + "theoretical_loss": 3.4460798124591947, + "tokens_seen": 1894055936 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043337373737373735, + "loss": 1.377, + "theoretical_loss": 3.446069764213101, + "tokens_seen": 1894121472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333535353535354, + "loss": 1.3279, + "theoretical_loss": 3.4460597164120097, + "tokens_seen": 1894187008 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.36241447925567627, + "objective/train/avg_token_score": 0.02387678064405918, + "objective/train/docs_used": 1070823, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2387202978134155, + "objective/train/original_loss": 4.809488296508789, + "objective/train/theoretical_loss": 3.4460496690558866, + "objective/train/tokens_used": 253111776, + "objective/train/unlikelihood_loss": 0.04541734606027603, + "theoretical_loss": 3.4460496690558866, + "tokens_seen": 1894252544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043333333333333337, + "loss": 1.2874, + "theoretical_loss": 3.4460496690558866, + "tokens_seen": 1894252544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333131313131313, + "loss": 1.5159, + "theoretical_loss": 3.4460396221446956, + "tokens_seen": 1894318080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332929292929293, + "loss": 1.2886, + "theoretical_loss": 3.4460295756784025, + "tokens_seen": 1894383616 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043327272727272724, + "loss": 1.3834, + "theoretical_loss": 3.446019529656972, + "tokens_seen": 1894449152 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332525252525253, + "loss": 1.3738, + "theoretical_loss": 3.446009484080368, + "tokens_seen": 1894514688 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043323232323232326, + "loss": 1.4346, + "theoretical_loss": 3.445999438948557, + "tokens_seen": 1894580224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332121212121212, + "loss": 1.415, + "theoretical_loss": 3.4459893942615025, + "tokens_seen": 1894645760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331919191919192, + "loss": 1.3809, + "theoretical_loss": 3.4459793500191704, + "tokens_seen": 1894711296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331717171717172, + "loss": 1.5446, + "theoretical_loss": 3.4459693062215253, + "tokens_seen": 1894776832 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331515151515152, + "loss": 1.5414, + "theoretical_loss": 3.445959262868532, + "tokens_seen": 1894842368 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043313131313131315, + "loss": 1.1756, + "theoretical_loss": 3.4459492199601556, + "tokens_seen": 1894907904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331111111111111, + "loss": 1.5137, + "theoretical_loss": 3.445939177496361, + "tokens_seen": 1894973440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043309090909090907, + "loss": 1.2745, + "theoretical_loss": 3.445929135477113, + "tokens_seen": 1895038976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330707070707071, + "loss": 1.6113, + "theoretical_loss": 3.445919093902377, + "tokens_seen": 1895104512 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330505050505051, + "loss": 1.4817, + "theoretical_loss": 3.4459090527721177, + "tokens_seen": 1895170048 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043303030303030305, + "loss": 1.6125, + "theoretical_loss": 3.4458990120863, + "tokens_seen": 1895235584 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433010101010101, + "loss": 1.345, + "theoretical_loss": 3.445888971844888, + "tokens_seen": 1895301120 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432989898989899, + "loss": 1.4279, + "theoretical_loss": 3.4458789320478487, + "tokens_seen": 1895366656 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043296969696969697, + "loss": 1.4733, + "theoretical_loss": 3.4458688926951453, + "tokens_seen": 1895432192 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432949494949495, + "loss": 1.2651, + "theoretical_loss": 3.4458588537867434, + "tokens_seen": 1895497728 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043292929292929294, + "loss": 1.2785, + "theoretical_loss": 3.4458488153226083, + "tokens_seen": 1895563264 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329090909090909, + "loss": 1.3361, + "theoretical_loss": 3.445838777302704, + "tokens_seen": 1895628800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328888888888889, + "loss": 1.4636, + "theoretical_loss": 3.4458287397269967, + "tokens_seen": 1895694336 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043286868686868686, + "loss": 1.257, + "theoretical_loss": 3.4458187025954503, + "tokens_seen": 1895759872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328484848484849, + "loss": 1.486, + "theoretical_loss": 3.4458086659080305, + "tokens_seen": 1895825408 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.3908846378326416, + "objective/train/avg_token_score": 0.010636522434651852, + "objective/train/docs_used": 1071554, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2562838792800903, + "objective/train/original_loss": 4.368358612060547, + "objective/train/theoretical_loss": 3.4457986296647025, + "objective/train/tokens_used": 254750176, + "objective/train/unlikelihood_loss": 0.06785128265619278, + "theoretical_loss": 3.4457986296647025, + "tokens_seen": 1895890944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043282828282828283, + "loss": 1.4474, + "theoretical_loss": 3.4457986296647025, + "tokens_seen": 1895890944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043280808080808084, + "loss": 1.3487, + "theoretical_loss": 3.4457885938654305, + "tokens_seen": 1895956480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327878787878788, + "loss": 1.4548, + "theoretical_loss": 3.44577855851018, + "tokens_seen": 1896022016 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043276767676767676, + "loss": 1.3746, + "theoretical_loss": 3.4457685235989155, + "tokens_seen": 1896087552 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043274747474747477, + "loss": 1.4116, + "theoretical_loss": 3.445758489131603, + "tokens_seen": 1896153088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327272727272727, + "loss": 1.5319, + "theoretical_loss": 3.4457484551082063, + "tokens_seen": 1896218624 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043270707070707073, + "loss": 1.5303, + "theoretical_loss": 3.4457384215286915, + "tokens_seen": 1896284160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326868686868687, + "loss": 1.2872, + "theoretical_loss": 3.4457283883930234, + "tokens_seen": 1896349696 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043266666666666665, + "loss": 1.2291, + "theoretical_loss": 3.4457183557011666, + "tokens_seen": 1896415232 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043264646464646466, + "loss": 1.1889, + "theoretical_loss": 3.4457083234530863, + "tokens_seen": 1896480768 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043262626262626267, + "loss": 1.3665, + "theoretical_loss": 3.445698291648747, + "tokens_seen": 1896546304 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043260606060606063, + "loss": 1.3058, + "theoretical_loss": 3.445688260288115, + "tokens_seen": 1896611840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325858585858586, + "loss": 1.4865, + "theoretical_loss": 3.445678229371154, + "tokens_seen": 1896677376 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043256565656565654, + "loss": 1.4227, + "theoretical_loss": 3.4456681988978306, + "tokens_seen": 1896742912 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043254545454545455, + "loss": 1.0513, + "theoretical_loss": 3.4456581688681087, + "tokens_seen": 1896808448 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043252525252525256, + "loss": 1.4629, + "theoretical_loss": 3.4456481392819533, + "tokens_seen": 1896873984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325050505050505, + "loss": 1.2892, + "theoretical_loss": 3.44563811013933, + "tokens_seen": 1896939520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324848484848485, + "loss": 1.5847, + "theoretical_loss": 3.445628081440203, + "tokens_seen": 1897005056 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043246464646464643, + "loss": 1.2676, + "theoretical_loss": 3.445618053184539, + "tokens_seen": 1897070592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324444444444445, + "loss": 1.4628, + "theoretical_loss": 3.4456080253723016, + "tokens_seen": 1897136128 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043242424242424246, + "loss": 1.5347, + "theoretical_loss": 3.445597998003456, + "tokens_seen": 1897201664 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324040404040404, + "loss": 1.38, + "theoretical_loss": 3.445587971077968, + "tokens_seen": 1897267200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043238383838383837, + "loss": 1.2716, + "theoretical_loss": 3.4455779445958026, + "tokens_seen": 1897332736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323636363636363, + "loss": 1.4612, + "theoretical_loss": 3.4455679185569243, + "tokens_seen": 1897398272 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323434343434344, + "loss": 1.404, + "theoretical_loss": 3.4455578929612987, + "tokens_seen": 1897463808 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.2730327546596527, + "objective/train/avg_token_score": 0.044983237981796265, + "objective/train/docs_used": 1072893, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9510472416877747, + "objective/train/original_loss": 5.534091949462891, + "objective/train/theoretical_loss": 3.44554786780889, + "objective/train/tokens_used": 256388576, + "objective/train/unlikelihood_loss": 0.05126022920012474, + "theoretical_loss": 3.44554786780889, + "tokens_seen": 1897529344 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043232323232323235, + "loss": 1.0224, + "theoretical_loss": 3.44554786780889, + "tokens_seen": 1897529344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323030303030303, + "loss": 1.3145, + "theoretical_loss": 3.4455378430996646, + "tokens_seen": 1897594880 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043228282828282826, + "loss": 1.354, + "theoretical_loss": 3.445527818833587, + "tokens_seen": 1897660416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322626262626262, + "loss": 1.3433, + "theoretical_loss": 3.4455177950106215, + "tokens_seen": 1897725952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322424242424243, + "loss": 1.1633, + "theoretical_loss": 3.445507771630735, + "tokens_seen": 1897791488 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043222222222222224, + "loss": 1.7054, + "theoretical_loss": 3.4454977486938914, + "tokens_seen": 1897857024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322020202020202, + "loss": 1.3594, + "theoretical_loss": 3.4454877262000556, + "tokens_seen": 1897922560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043218181818181816, + "loss": 1.3057, + "theoretical_loss": 3.4454777041491935, + "tokens_seen": 1897988096 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043216161616161617, + "loss": 1.1239, + "theoretical_loss": 3.44546768254127, + "tokens_seen": 1898053632 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321414141414142, + "loss": 1.3014, + "theoretical_loss": 3.44545766137625, + "tokens_seen": 1898119168 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043212121212121213, + "loss": 1.5642, + "theoretical_loss": 3.4454476406540984, + "tokens_seen": 1898184704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321010101010101, + "loss": 1.1932, + "theoretical_loss": 3.445437620374781, + "tokens_seen": 1898250240 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043208080808080805, + "loss": 1.3947, + "theoretical_loss": 3.445427600538263, + "tokens_seen": 1898315776 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043206060606060606, + "loss": 1.2837, + "theoretical_loss": 3.4454175811445085, + "tokens_seen": 1898381312 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043204040404040407, + "loss": 1.4581, + "theoretical_loss": 3.445407562193484, + "tokens_seen": 1898446848 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432020202020202, + "loss": 1.3282, + "theoretical_loss": 3.4453975436851536, + "tokens_seen": 1898512384 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432, + "loss": 1.2793, + "theoretical_loss": 3.445387525619483, + "tokens_seen": 1898577920 + }, + { + "epoch": 0.14, + "learning_rate": 0.000431979797979798, + "loss": 1.1152, + "theoretical_loss": 3.445377507996437, + "tokens_seen": 1898643456 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043195959595959595, + "loss": 1.4725, + "theoretical_loss": 3.445367490815981, + "tokens_seen": 1898708992 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043193939393939396, + "loss": 1.2346, + "theoretical_loss": 3.4453574740780804, + "tokens_seen": 1898774528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319191919191919, + "loss": 1.4218, + "theoretical_loss": 3.4453474577826997, + "tokens_seen": 1898840064 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318989898989899, + "loss": 1.2056, + "theoretical_loss": 3.445337441929805, + "tokens_seen": 1898905600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318787878787879, + "loss": 1.2807, + "theoretical_loss": 3.445327426519361, + "tokens_seen": 1898971136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318585858585859, + "loss": 1.4656, + "theoretical_loss": 3.4453174115513323, + "tokens_seen": 1899036672 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043183838383838386, + "loss": 1.4633, + "theoretical_loss": 3.445307397025685, + "tokens_seen": 1899102208 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.42344820499420166, + "objective/train/avg_token_score": 0.01560823805630207, + "objective/train/docs_used": 1073632, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4783105850219727, + "objective/train/original_loss": 4.294405937194824, + "objective/train/theoretical_loss": 3.445297382942384, + "objective/train/tokens_used": 258026976, + "objective/train/unlikelihood_loss": 0.07553756982088089, + "theoretical_loss": 3.445297382942384, + "tokens_seen": 1899167744 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318181818181818, + "loss": 1.2705, + "theoretical_loss": 3.445297382942384, + "tokens_seen": 1899167744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317979797979798, + "loss": 1.622, + "theoretical_loss": 3.4452873693013943, + "tokens_seen": 1899233280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317777777777778, + "loss": 1.4875, + "theoretical_loss": 3.4452773561026815, + "tokens_seen": 1899298816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317575757575758, + "loss": 1.5592, + "theoretical_loss": 3.4452673433462104, + "tokens_seen": 1899364352 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043173737373737375, + "loss": 1.3319, + "theoretical_loss": 3.4452573310319465, + "tokens_seen": 1899429888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317171717171717, + "loss": 1.4507, + "theoretical_loss": 3.4452473191598547, + "tokens_seen": 1899495424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316969696969697, + "loss": 1.4265, + "theoretical_loss": 3.4452373077299003, + "tokens_seen": 1899560960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316767676767677, + "loss": 1.4393, + "theoretical_loss": 3.4452272967420488, + "tokens_seen": 1899626496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316565656565657, + "loss": 1.5585, + "theoretical_loss": 3.4452172861962653, + "tokens_seen": 1899692032 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043163636363636364, + "loss": 1.3088, + "theoretical_loss": 3.445207276092515, + "tokens_seen": 1899757568 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043161616161616165, + "loss": 1.4769, + "theoretical_loss": 3.445197266430763, + "tokens_seen": 1899823104 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315959595959596, + "loss": 1.2856, + "theoretical_loss": 3.4451872572109745, + "tokens_seen": 1899888640 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043157575757575757, + "loss": 1.4165, + "theoretical_loss": 3.445177248433115, + "tokens_seen": 1899954176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315555555555556, + "loss": 1.5391, + "theoretical_loss": 3.44516724009715, + "tokens_seen": 1900019712 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043153535353535353, + "loss": 1.4838, + "theoretical_loss": 3.445157232203044, + "tokens_seen": 1900085248 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043151515151515154, + "loss": 1.2202, + "theoretical_loss": 3.4451472247507624, + "tokens_seen": 1900150784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314949494949495, + "loss": 1.3836, + "theoretical_loss": 3.445137217740271, + "tokens_seen": 1900216320 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043147474747474746, + "loss": 1.0373, + "theoretical_loss": 3.4451272111715348, + "tokens_seen": 1900281856 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043145454545454547, + "loss": 1.2555, + "theoretical_loss": 3.445117205044519, + "tokens_seen": 1900347392 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314343434343435, + "loss": 1.3939, + "theoretical_loss": 3.445107199359189, + "tokens_seen": 1900412928 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043141414141414144, + "loss": 1.4978, + "theoretical_loss": 3.44509719411551, + "tokens_seen": 1900478464 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313939393939394, + "loss": 1.5079, + "theoretical_loss": 3.4450871893134467, + "tokens_seen": 1900544000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043137373737373735, + "loss": 1.2719, + "theoretical_loss": 3.4450771849529653, + "tokens_seen": 1900609536 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043135353535353536, + "loss": 1.3437, + "theoretical_loss": 3.4450671810340308, + "tokens_seen": 1900675072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313333333333334, + "loss": 1.3644, + "theoretical_loss": 3.4450571775566083, + "tokens_seen": 1900740608 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.41379520297050476, + "objective/train/avg_token_score": 0.014362385496497154, + "objective/train/docs_used": 1075027, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3654704093933105, + "objective/train/original_loss": 4.367673397064209, + "objective/train/theoretical_loss": 3.445047174520663, + "objective/train/tokens_used": 259665376, + "objective/train/unlikelihood_loss": 0.07200692594051361, + "theoretical_loss": 3.445047174520663, + "tokens_seen": 1900806144 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043131313131313133, + "loss": 1.5024, + "theoretical_loss": 3.445047174520663, + "tokens_seen": 1900806144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312929292929293, + "loss": 1.1957, + "theoretical_loss": 3.4450371719261605, + "tokens_seen": 1900871680 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043127272727272724, + "loss": 1.3585, + "theoretical_loss": 3.4450271697730663, + "tokens_seen": 1900937216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312525252525253, + "loss": 1.4242, + "theoretical_loss": 3.445017168061345, + "tokens_seen": 1901002752 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043123232323232327, + "loss": 1.5416, + "theoretical_loss": 3.4450071667909623, + "tokens_seen": 1901068288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312121212121212, + "loss": 1.5098, + "theoretical_loss": 3.4449971659618837, + "tokens_seen": 1901133824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311919191919192, + "loss": 1.3824, + "theoretical_loss": 3.4449871655740743, + "tokens_seen": 1901199360 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043117171717171714, + "loss": 1.3807, + "theoretical_loss": 3.444977165627499, + "tokens_seen": 1901264896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311515151515152, + "loss": 1.4715, + "theoretical_loss": 3.444967166122124, + "tokens_seen": 1901330432 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043113131313131316, + "loss": 1.228, + "theoretical_loss": 3.444957167057914, + "tokens_seen": 1901395968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311111111111111, + "loss": 1.4939, + "theoretical_loss": 3.444947168434835, + "tokens_seen": 1901461504 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043109090909090907, + "loss": 1.4963, + "theoretical_loss": 3.4449371702528513, + "tokens_seen": 1901527040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310707070707071, + "loss": 1.336, + "theoretical_loss": 3.444927172511929, + "tokens_seen": 1901592576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310505050505051, + "loss": 1.2116, + "theoretical_loss": 3.4449171752120336, + "tokens_seen": 1901658112 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043103030303030305, + "loss": 1.2582, + "theoretical_loss": 3.4449071783531293, + "tokens_seen": 1901723648 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431010101010101, + "loss": 1.5321, + "theoretical_loss": 3.4448971819351826, + "tokens_seen": 1901789184 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043098989898989897, + "loss": 1.5561, + "theoretical_loss": 3.4448871859581587, + "tokens_seen": 1901854720 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430969696969697, + "loss": 1.478, + "theoretical_loss": 3.444877190422023, + "tokens_seen": 1901920256 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430949494949495, + "loss": 1.5553, + "theoretical_loss": 3.44486719532674, + "tokens_seen": 1901985792 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043092929292929294, + "loss": 1.5061, + "theoretical_loss": 3.444857200672276, + "tokens_seen": 1902051328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309090909090909, + "loss": 1.4788, + "theoretical_loss": 3.4448472064585958, + "tokens_seen": 1902116864 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308888888888889, + "loss": 1.5469, + "theoretical_loss": 3.4448372126856652, + "tokens_seen": 1902182400 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043086868686868687, + "loss": 1.4974, + "theoretical_loss": 3.4448272193534493, + "tokens_seen": 1902247936 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308484848484849, + "loss": 1.3048, + "theoretical_loss": 3.4448172264619137, + "tokens_seen": 1902313472 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043082828282828284, + "loss": 1.5208, + "theoretical_loss": 3.444807234011024, + "tokens_seen": 1902379008 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.4122678339481354, + "objective/train/avg_token_score": 0.011399907991290092, + "objective/train/docs_used": 1075669, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.282902479171753, + "objective/train/original_loss": 4.224909782409668, + "objective/train/theoretical_loss": 3.4447972420007447, + "objective/train/tokens_used": 261303776, + "objective/train/unlikelihood_loss": 0.0683131068944931, + "theoretical_loss": 3.4447972420007447, + "tokens_seen": 1902444544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308080808080808, + "loss": 1.3168, + "theoretical_loss": 3.4447972420007447, + "tokens_seen": 1902444544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307878787878788, + "loss": 1.4082, + "theoretical_loss": 3.444787250431042, + "tokens_seen": 1902510080 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043076767676767676, + "loss": 1.5043, + "theoretical_loss": 3.444777259301881, + "tokens_seen": 1902575616 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043074747474747477, + "loss": 1.6162, + "theoretical_loss": 3.4447672686132274, + "tokens_seen": 1902641152 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043072727272727273, + "loss": 1.3416, + "theoretical_loss": 3.4447572783650457, + "tokens_seen": 1902706688 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043070707070707074, + "loss": 1.2339, + "theoretical_loss": 3.444747288557303, + "tokens_seen": 1902772224 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306868686868687, + "loss": 1.5255, + "theoretical_loss": 3.444737299189963, + "tokens_seen": 1902837760 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043066666666666665, + "loss": 1.3823, + "theoretical_loss": 3.444727310262992, + "tokens_seen": 1902903296 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043064646464646467, + "loss": 1.3543, + "theoretical_loss": 3.444717321776355, + "tokens_seen": 1902968832 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306262626262626, + "loss": 1.4874, + "theoretical_loss": 3.444707333730018, + "tokens_seen": 1903034368 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043060606060606063, + "loss": 1.5816, + "theoretical_loss": 3.444697346123946, + "tokens_seen": 1903099904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305858585858586, + "loss": 1.3016, + "theoretical_loss": 3.444687358958104, + "tokens_seen": 1903165440 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043056565656565655, + "loss": 1.3505, + "theoretical_loss": 3.4446773722324586, + "tokens_seen": 1903230976 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043054545454545456, + "loss": 1.4311, + "theoretical_loss": 3.4446673859469747, + "tokens_seen": 1903296512 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043052525252525257, + "loss": 1.5355, + "theoretical_loss": 3.444657400101617, + "tokens_seen": 1903362048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305050505050505, + "loss": 1.3983, + "theoretical_loss": 3.444647414696352, + "tokens_seen": 1903427584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304848484848485, + "loss": 1.2973, + "theoretical_loss": 3.444637429731145, + "tokens_seen": 1903493120 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043046464646464644, + "loss": 1.5914, + "theoretical_loss": 3.4446274452059606, + "tokens_seen": 1903558656 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043044444444444445, + "loss": 1.3239, + "theoretical_loss": 3.444617461120765, + "tokens_seen": 1903624192 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043042424242424246, + "loss": 1.2012, + "theoretical_loss": 3.444607477475524, + "tokens_seen": 1903689728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304040404040404, + "loss": 1.4485, + "theoretical_loss": 3.4445974942702025, + "tokens_seen": 1903755264 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303838383838384, + "loss": 1.4015, + "theoretical_loss": 3.4445875115047655, + "tokens_seen": 1903820800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043036363636363633, + "loss": 1.5879, + "theoretical_loss": 3.4445775291791794, + "tokens_seen": 1903886336 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043034343434343434, + "loss": 1.4073, + "theoretical_loss": 3.444567547293409, + "tokens_seen": 1903951872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043032323232323235, + "loss": 1.3852, + "theoretical_loss": 3.444557565847421, + "tokens_seen": 1904017408 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.3672409653663635, + "objective/train/avg_token_score": 0.005817563273012638, + "objective/train/docs_used": 1076839, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2039426565170288, + "objective/train/original_loss": 3.8227851390838623, + "objective/train/theoretical_loss": 3.444547584841179, + "objective/train/tokens_used": 262942176, + "objective/train/unlikelihood_loss": 0.13508202135562897, + "theoretical_loss": 3.444547584841179, + "tokens_seen": 1904082944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303030303030303, + "loss": 1.2922, + "theoretical_loss": 3.444547584841179, + "tokens_seen": 1904082944 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043028282828282827, + "loss": 1.2261, + "theoretical_loss": 3.4445376042746503, + "tokens_seen": 1904148480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302626262626262, + "loss": 1.3418, + "theoretical_loss": 3.444527624147799, + "tokens_seen": 1904214016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302424242424243, + "loss": 1.402, + "theoretical_loss": 3.444517644460591, + "tokens_seen": 1904279552 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043022222222222225, + "loss": 1.406, + "theoretical_loss": 3.4445076652129925, + "tokens_seen": 1904345088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302020202020202, + "loss": 1.3695, + "theoretical_loss": 3.4444976864049686, + "tokens_seen": 1904410624 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043018181818181816, + "loss": 1.5536, + "theoretical_loss": 3.444487708036484, + "tokens_seen": 1904476160 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043016161616161617, + "loss": 1.4343, + "theoretical_loss": 3.444477730107505, + "tokens_seen": 1904541696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301414141414142, + "loss": 1.1865, + "theoretical_loss": 3.4444677526179976, + "tokens_seen": 1904607232 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043012121212121214, + "loss": 1.6126, + "theoretical_loss": 3.444457775567927, + "tokens_seen": 1904672768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301010101010101, + "loss": 1.6826, + "theoretical_loss": 3.444447798957258, + "tokens_seen": 1904738304 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043008080808080805, + "loss": 1.335, + "theoretical_loss": 3.4444378227859564, + "tokens_seen": 1904803840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300606060606061, + "loss": 1.4046, + "theoretical_loss": 3.4444278470539884, + "tokens_seen": 1904869376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300404040404041, + "loss": 1.5168, + "theoretical_loss": 3.4444178717613188, + "tokens_seen": 1904934912 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043002020202020203, + "loss": 1.4559, + "theoretical_loss": 3.4444078969079133, + "tokens_seen": 1905000448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043, + "loss": 1.3272, + "theoretical_loss": 3.4443979224937378, + "tokens_seen": 1905065984 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042997979797979795, + "loss": 1.3365, + "theoretical_loss": 3.4443879485187576, + "tokens_seen": 1905131520 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429959595959596, + "loss": 1.4219, + "theoretical_loss": 3.4443779749829386, + "tokens_seen": 1905197056 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042993939393939397, + "loss": 1.5434, + "theoretical_loss": 3.4443680018862457, + "tokens_seen": 1905262592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004299191919191919, + "loss": 1.5114, + "theoretical_loss": 3.4443580292286446, + "tokens_seen": 1905328128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298989898989899, + "loss": 1.5505, + "theoretical_loss": 3.4443480570101017, + "tokens_seen": 1905393664 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298787878787879, + "loss": 1.3701, + "theoretical_loss": 3.444338085230582, + "tokens_seen": 1905459200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298585858585859, + "loss": 1.3049, + "theoretical_loss": 3.44432811389005, + "tokens_seen": 1905524736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042983838383838386, + "loss": 1.4485, + "theoretical_loss": 3.444318142988473, + "tokens_seen": 1905590272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298181818181818, + "loss": 1.4768, + "theoretical_loss": 3.444308172525816, + "tokens_seen": 1905655808 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.35905423760414124, + "objective/train/avg_token_score": 0.026558930054306984, + "objective/train/docs_used": 1077487, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1562389135360718, + "objective/train/original_loss": 4.9117536544799805, + "objective/train/theoretical_loss": 3.444298202502044, + "objective/train/tokens_used": 264580576, + "objective/train/unlikelihood_loss": 0.038373254239559174, + "theoretical_loss": 3.444298202502044, + "tokens_seen": 1905721344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297979797979798, + "loss": 1.3541, + "theoretical_loss": 3.444298202502044, + "tokens_seen": 1905721344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297777777777778, + "loss": 1.3404, + "theoretical_loss": 3.444288232917123, + "tokens_seen": 1905786880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297575757575758, + "loss": 1.3965, + "theoretical_loss": 3.444278263771019, + "tokens_seen": 1905852416 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042973737373737375, + "loss": 1.3603, + "theoretical_loss": 3.444268295063697, + "tokens_seen": 1905917952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297171717171717, + "loss": 1.5391, + "theoretical_loss": 3.444258326795123, + "tokens_seen": 1905983488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296969696969697, + "loss": 1.5438, + "theoretical_loss": 3.444248358965263, + "tokens_seen": 1906049024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296767676767677, + "loss": 1.3518, + "theoretical_loss": 3.444238391574081, + "tokens_seen": 1906114560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296565656565657, + "loss": 1.1733, + "theoretical_loss": 3.4442284246215444, + "tokens_seen": 1906180096 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042963636363636365, + "loss": 1.4349, + "theoretical_loss": 3.444218458107618, + "tokens_seen": 1906245632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296161616161616, + "loss": 1.436, + "theoretical_loss": 3.444208492032267, + "tokens_seen": 1906311168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295959595959596, + "loss": 1.3842, + "theoretical_loss": 3.444198526395458, + "tokens_seen": 1906376704 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042957575757575757, + "loss": 1.3577, + "theoretical_loss": 3.444188561197156, + "tokens_seen": 1906442240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295555555555556, + "loss": 1.4073, + "theoretical_loss": 3.4441785964373266, + "tokens_seen": 1906507776 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042953535353535354, + "loss": 1.3905, + "theoretical_loss": 3.444168632115936, + "tokens_seen": 1906573312 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042951515151515155, + "loss": 1.3146, + "theoretical_loss": 3.444158668232949, + "tokens_seen": 1906638848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004294949494949495, + "loss": 1.4867, + "theoretical_loss": 3.444148704788332, + "tokens_seen": 1906704384 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042947474747474746, + "loss": 1.5472, + "theoretical_loss": 3.44413874178205, + "tokens_seen": 1906769920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004294545454545455, + "loss": 1.4212, + "theoretical_loss": 3.4441287792140693, + "tokens_seen": 1906835456 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042943434343434343, + "loss": 1.2909, + "theoretical_loss": 3.4441188170843553, + "tokens_seen": 1906900992 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042941414141414144, + "loss": 1.4079, + "theoretical_loss": 3.444108855392874, + "tokens_seen": 1906966528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293939393939394, + "loss": 1.4177, + "theoretical_loss": 3.44409889413959, + "tokens_seen": 1907032064 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042937373737373736, + "loss": 1.4536, + "theoretical_loss": 3.4440889333244695, + "tokens_seen": 1907097600 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042935353535353537, + "loss": 1.4919, + "theoretical_loss": 3.444078972947479, + "tokens_seen": 1907163136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293333333333334, + "loss": 1.3301, + "theoretical_loss": 3.4440690130085825, + "tokens_seen": 1907228672 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042931313131313134, + "loss": 1.3173, + "theoretical_loss": 3.4440590535077478, + "tokens_seen": 1907294208 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.3209921717643738, + "objective/train/avg_token_score": 0.06907281279563904, + "objective/train/docs_used": 1078017, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0279090404510498, + "objective/train/original_loss": 4.60770320892334, + "objective/train/theoretical_loss": 3.4440490944449387, + "objective/train/tokens_used": 266218976, + "objective/train/unlikelihood_loss": 0.046941839158535004, + "theoretical_loss": 3.4440490944449387, + "tokens_seen": 1907359744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292929292929293, + "loss": 1.2658, + "theoretical_loss": 3.4440490944449387, + "tokens_seen": 1907359744 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042927272727272725, + "loss": 1.43, + "theoretical_loss": 3.444039135820122, + "tokens_seen": 1907425280 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042925252525252526, + "loss": 1.4694, + "theoretical_loss": 3.444029177633263, + "tokens_seen": 1907490816 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042923232323232327, + "loss": 1.2123, + "theoretical_loss": 3.4440192198843276, + "tokens_seen": 1907556352 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042921212121212123, + "loss": 1.3888, + "theoretical_loss": 3.4440092625732808, + "tokens_seen": 1907621888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291919191919192, + "loss": 1.3047, + "theoretical_loss": 3.443999305700089, + "tokens_seen": 1907687424 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042917171717171714, + "loss": 1.2473, + "theoretical_loss": 3.443989349264718, + "tokens_seen": 1907752960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291515151515152, + "loss": 1.4243, + "theoretical_loss": 3.4439793932671323, + "tokens_seen": 1907818496 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042913131313131316, + "loss": 1.4856, + "theoretical_loss": 3.4439694377073, + "tokens_seen": 1907884032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291111111111111, + "loss": 1.5347, + "theoretical_loss": 3.4439594825851843, + "tokens_seen": 1907949568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290909090909091, + "loss": 1.5656, + "theoretical_loss": 3.4439495279007524, + "tokens_seen": 1908015104 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042907070707070703, + "loss": 1.3704, + "theoretical_loss": 3.443939573653969, + "tokens_seen": 1908080640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290505050505051, + "loss": 1.2791, + "theoretical_loss": 3.443929619844801, + "tokens_seen": 1908146176 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042903030303030306, + "loss": 1.1324, + "theoretical_loss": 3.4439196664732137, + "tokens_seen": 1908211712 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429010101010101, + "loss": 1.5249, + "theoretical_loss": 3.4439097135391723, + "tokens_seen": 1908277248 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042898989898989897, + "loss": 1.5048, + "theoretical_loss": 3.443899761042643, + "tokens_seen": 1908342784 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042896969696969693, + "loss": 1.5334, + "theoretical_loss": 3.4438898089835916, + "tokens_seen": 1908408320 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428949494949495, + "loss": 1.6011, + "theoretical_loss": 3.443879857361984, + "tokens_seen": 1908473856 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042892929292929295, + "loss": 1.4087, + "theoretical_loss": 3.4438699061777855, + "tokens_seen": 1908539392 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289090909090909, + "loss": 1.3535, + "theoretical_loss": 3.4438599554309617, + "tokens_seen": 1908604928 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042888888888888886, + "loss": 1.4036, + "theoretical_loss": 3.4438500051214787, + "tokens_seen": 1908670464 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288686868686869, + "loss": 1.2635, + "theoretical_loss": 3.4438400552493027, + "tokens_seen": 1908736000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288484848484849, + "loss": 1.3974, + "theoretical_loss": 3.4438301058143987, + "tokens_seen": 1908801536 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042882828282828284, + "loss": 1.5568, + "theoretical_loss": 3.443820156816733, + "tokens_seen": 1908867072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288080808080808, + "loss": 1.4838, + "theoretical_loss": 3.443810208256271, + "tokens_seen": 1908932608 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.37292277812957764, + "objective/train/avg_token_score": 0.02143978700041771, + "objective/train/docs_used": 1079511, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.331836462020874, + "objective/train/original_loss": 4.501890659332275, + "objective/train/theoretical_loss": 3.4438002601329787, + "objective/train/tokens_used": 267857376, + "objective/train/unlikelihood_loss": 0.05701581388711929, + "theoretical_loss": 3.4438002601329787, + "tokens_seen": 1908998144 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042878787878787876, + "loss": 1.5142, + "theoretical_loss": 3.4438002601329787, + "tokens_seen": 1908998144 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042876767676767677, + "loss": 1.4552, + "theoretical_loss": 3.443790312446822, + "tokens_seen": 1909063680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287474747474748, + "loss": 1.1977, + "theoretical_loss": 3.4437803651977665, + "tokens_seen": 1909129216 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042872727272727274, + "loss": 1.5264, + "theoretical_loss": 3.4437704183857782, + "tokens_seen": 1909194752 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287070707070707, + "loss": 1.3258, + "theoretical_loss": 3.443760472010822, + "tokens_seen": 1909260288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286868686868687, + "loss": 1.3133, + "theoretical_loss": 3.443750526072865, + "tokens_seen": 1909325824 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042866666666666666, + "loss": 1.2519, + "theoretical_loss": 3.4437405805718724, + "tokens_seen": 1909391360 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042864646464646467, + "loss": 1.282, + "theoretical_loss": 3.4437306355078094, + "tokens_seen": 1909456896 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042862626262626263, + "loss": 1.3893, + "theoretical_loss": 3.4437206908806433, + "tokens_seen": 1909522432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286060606060606, + "loss": 1.2844, + "theoretical_loss": 3.4437107466903383, + "tokens_seen": 1909587968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285858585858586, + "loss": 1.5531, + "theoretical_loss": 3.443700802936861, + "tokens_seen": 1909653504 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042856565656565655, + "loss": 1.5443, + "theoretical_loss": 3.443690859620178, + "tokens_seen": 1909719040 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042854545454545456, + "loss": 1.4003, + "theoretical_loss": 3.443680916740253, + "tokens_seen": 1909784576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285252525252525, + "loss": 1.4069, + "theoretical_loss": 3.443670974297054, + "tokens_seen": 1909850112 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042850505050505053, + "loss": 1.574, + "theoretical_loss": 3.443661032290546, + "tokens_seen": 1909915648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284848484848485, + "loss": 1.5845, + "theoretical_loss": 3.4436510907206945, + "tokens_seen": 1909981184 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042846464646464645, + "loss": 1.2667, + "theoretical_loss": 3.4436411495874655, + "tokens_seen": 1910046720 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042844444444444446, + "loss": 1.1777, + "theoretical_loss": 3.4436312088908254, + "tokens_seen": 1910112256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284242424242424, + "loss": 1.3442, + "theoretical_loss": 3.4436212686307393, + "tokens_seen": 1910177792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284040404040404, + "loss": 1.3153, + "theoretical_loss": 3.4436113288071732, + "tokens_seen": 1910243328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283838383838384, + "loss": 1.5457, + "theoretical_loss": 3.4436013894200936, + "tokens_seen": 1910308864 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283636363636364, + "loss": 1.2672, + "theoretical_loss": 3.443591450469466, + "tokens_seen": 1910374400 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042834343434343435, + "loss": 1.084, + "theoretical_loss": 3.4435815119552555, + "tokens_seen": 1910439936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042832323232323236, + "loss": 1.3617, + "theoretical_loss": 3.443571573877429, + "tokens_seen": 1910505472 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283030303030303, + "loss": 1.3787, + "theoretical_loss": 3.443561636235952, + "tokens_seen": 1910571008 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.5610948204994202, + "objective/train/avg_token_score": 0.003200286766514182, + "objective/train/docs_used": 1080108, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.690970540046692, + "objective/train/original_loss": 3.2980706691741943, + "objective/train/theoretical_loss": 3.4435516990307904, + "objective/train/tokens_used": 269495776, + "objective/train/unlikelihood_loss": 0.08375748991966248, + "theoretical_loss": 3.4435516990307904, + "tokens_seen": 1910636544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282828282828283, + "loss": 1.4718, + "theoretical_loss": 3.4435516990307904, + "tokens_seen": 1910636544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282626262626263, + "loss": 1.5774, + "theoretical_loss": 3.44354176226191, + "tokens_seen": 1910702080 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042824242424242424, + "loss": 1.6015, + "theoretical_loss": 3.4435318259292766, + "tokens_seen": 1910767616 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042822222222222225, + "loss": 1.3461, + "theoretical_loss": 3.443521890032856, + "tokens_seen": 1910833152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282020202020202, + "loss": 1.394, + "theoretical_loss": 3.4435119545726147, + "tokens_seen": 1910898688 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042818181818181817, + "loss": 1.3769, + "theoretical_loss": 3.443502019548518, + "tokens_seen": 1910964224 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281616161616162, + "loss": 1.1004, + "theoretical_loss": 3.4434920849605324, + "tokens_seen": 1911029760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281414141414142, + "loss": 1.5169, + "theoretical_loss": 3.443482150808623, + "tokens_seen": 1911095296 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042812121212121215, + "loss": 1.4129, + "theoretical_loss": 3.443472217092756, + "tokens_seen": 1911160832 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281010101010101, + "loss": 1.3754, + "theoretical_loss": 3.443462283812898, + "tokens_seen": 1911226368 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042808080808080806, + "loss": 1.3636, + "theoretical_loss": 3.4434523509690136, + "tokens_seen": 1911291904 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042806060606060607, + "loss": 1.2775, + "theoretical_loss": 3.44344241856107, + "tokens_seen": 1911357440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280404040404041, + "loss": 1.6295, + "theoretical_loss": 3.4434324865890322, + "tokens_seen": 1911422976 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042802020202020204, + "loss": 1.5199, + "theoretical_loss": 3.4434225550528668, + "tokens_seen": 1911488512 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428, + "loss": 1.3486, + "theoretical_loss": 3.443412623952539, + "tokens_seen": 1911554048 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042797979797979795, + "loss": 1.4287, + "theoretical_loss": 3.4434026932880153, + "tokens_seen": 1911619584 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427959595959596, + "loss": 1.4952, + "theoretical_loss": 3.4433927630592613, + "tokens_seen": 1911685120 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427939393939394, + "loss": 1.6055, + "theoretical_loss": 3.4433828332662437, + "tokens_seen": 1911750656 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042791919191919193, + "loss": 1.3703, + "theoretical_loss": 3.4433729039089274, + "tokens_seen": 1911816192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278989898989899, + "loss": 1.3665, + "theoretical_loss": 3.443362974987279, + "tokens_seen": 1911881728 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042787878787878784, + "loss": 1.3683, + "theoretical_loss": 3.4433530465012643, + "tokens_seen": 1911947264 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278585858585859, + "loss": 1.3692, + "theoretical_loss": 3.443343118450849, + "tokens_seen": 1912012800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042783838383838387, + "loss": 1.4151, + "theoretical_loss": 3.4433331908359994, + "tokens_seen": 1912078336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278181818181818, + "loss": 1.4771, + "theoretical_loss": 3.4433232636566817, + "tokens_seen": 1912143872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277979797979798, + "loss": 1.4327, + "theoretical_loss": 3.443313336912861, + "tokens_seen": 1912209408 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.33864858746528625, + "objective/train/avg_token_score": 0.007881486788392067, + "objective/train/docs_used": 1081319, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1766334772109985, + "objective/train/original_loss": 4.45033597946167, + "objective/train/theoretical_loss": 3.443303410604504, + "objective/train/tokens_used": 271134176, + "objective/train/unlikelihood_loss": 0.05650121346116066, + "theoretical_loss": 3.443303410604504, + "tokens_seen": 1912274944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277777777777778, + "loss": 1.4087, + "theoretical_loss": 3.443303410604504, + "tokens_seen": 1912274944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277575757575758, + "loss": 1.6281, + "theoretical_loss": 3.4432934847315764, + "tokens_seen": 1912340480 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042773737373737376, + "loss": 1.4231, + "theoretical_loss": 3.443283559294044, + "tokens_seen": 1912406016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277171717171717, + "loss": 1.3221, + "theoretical_loss": 3.4432736342918737, + "tokens_seen": 1912471552 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276969696969697, + "loss": 1.3538, + "theoretical_loss": 3.44326370972503, + "tokens_seen": 1912537088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276767676767677, + "loss": 1.3519, + "theoretical_loss": 3.4432537855934804, + "tokens_seen": 1912602624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276565656565657, + "loss": 1.2999, + "theoretical_loss": 3.44324386189719, + "tokens_seen": 1912668160 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042763636363636365, + "loss": 1.4608, + "theoretical_loss": 3.4432339386361246, + "tokens_seen": 1912733696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276161616161616, + "loss": 1.5103, + "theoretical_loss": 3.4432240158102507, + "tokens_seen": 1912799232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275959595959596, + "loss": 1.3832, + "theoretical_loss": 3.443214093419534, + "tokens_seen": 1912864768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275757575757576, + "loss": 1.319, + "theoretical_loss": 3.4432041714639414, + "tokens_seen": 1912930304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275555555555556, + "loss": 1.2985, + "theoretical_loss": 3.4431942499434376, + "tokens_seen": 1912995840 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042753535353535354, + "loss": 1.2791, + "theoretical_loss": 3.4431843288579893, + "tokens_seen": 1913061376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275151515151515, + "loss": 1.3884, + "theoretical_loss": 3.443174408207563, + "tokens_seen": 1913126912 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274949494949495, + "loss": 1.3111, + "theoretical_loss": 3.4431644879921235, + "tokens_seen": 1913192448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042747474747474747, + "loss": 1.4369, + "theoretical_loss": 3.4431545682116376, + "tokens_seen": 1913257984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274545454545455, + "loss": 1.2989, + "theoretical_loss": 3.4431446488660713, + "tokens_seen": 1913323520 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042743434343434344, + "loss": 1.5759, + "theoretical_loss": 3.44313472995539, + "tokens_seen": 1913389056 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042741414141414145, + "loss": 1.5069, + "theoretical_loss": 3.443124811479561, + "tokens_seen": 1913454592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273939393939394, + "loss": 1.3957, + "theoretical_loss": 3.44311489343855, + "tokens_seen": 1913520128 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042737373737373736, + "loss": 1.4563, + "theoretical_loss": 3.443104975832322, + "tokens_seen": 1913585664 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273535353535354, + "loss": 1.1546, + "theoretical_loss": 3.4430950586608438, + "tokens_seen": 1913651200 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042733333333333333, + "loss": 1.3923, + "theoretical_loss": 3.4430851419240818, + "tokens_seen": 1913716736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042731313131313134, + "loss": 1.3045, + "theoretical_loss": 3.4430752256220014, + "tokens_seen": 1913782272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272929292929293, + "loss": 1.2177, + "theoretical_loss": 3.443065309754569, + "tokens_seen": 1913847808 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.29844820499420166, + "objective/train/avg_token_score": 0.011274569667875767, + "objective/train/docs_used": 1081978, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.975776731967926, + "objective/train/original_loss": 4.575207710266113, + "objective/train/theoretical_loss": 3.4430553943217506, + "objective/train/tokens_used": 272772576, + "objective/train/unlikelihood_loss": 0.051495835185050964, + "theoretical_loss": 3.4430553943217506, + "tokens_seen": 1913913344 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042727272727272726, + "loss": 1.4363, + "theoretical_loss": 3.4430553943217506, + "tokens_seen": 1913913344 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042725252525252527, + "loss": 1.283, + "theoretical_loss": 3.443045479323512, + "tokens_seen": 1913978880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272323232323233, + "loss": 1.5076, + "theoretical_loss": 3.4430355647598194, + "tokens_seen": 1914044416 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042721212121212123, + "loss": 1.2358, + "theoretical_loss": 3.4430256506306396, + "tokens_seen": 1914109952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271919191919192, + "loss": 1.4152, + "theoretical_loss": 3.443015736935938, + "tokens_seen": 1914175488 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042717171717171715, + "loss": 1.2006, + "theoretical_loss": 3.4430058236756804, + "tokens_seen": 1914241024 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042715151515151516, + "loss": 1.5529, + "theoretical_loss": 3.4429959108498336, + "tokens_seen": 1914306560 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042713131313131317, + "loss": 1.3738, + "theoretical_loss": 3.4429859984583633, + "tokens_seen": 1914372096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271111111111111, + "loss": 1.4871, + "theoretical_loss": 3.4429760865012353, + "tokens_seen": 1914437632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270909090909091, + "loss": 1.3976, + "theoretical_loss": 3.442966174978416, + "tokens_seen": 1914503168 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042707070707070704, + "loss": 1.1469, + "theoretical_loss": 3.4429562638898723, + "tokens_seen": 1914568704 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042705050505050505, + "loss": 1.2692, + "theoretical_loss": 3.442946353235569, + "tokens_seen": 1914634240 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042703030303030306, + "loss": 1.5126, + "theoretical_loss": 3.4429364430154727, + "tokens_seen": 1914699776 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427010101010101, + "loss": 1.4688, + "theoretical_loss": 3.44292653322955, + "tokens_seen": 1914765312 + }, + { + "epoch": 0.15, + "learning_rate": 0.000426989898989899, + "loss": 1.4204, + "theoretical_loss": 3.4429166238777658, + "tokens_seen": 1914830848 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042696969696969693, + "loss": 1.5229, + "theoretical_loss": 3.442906714960088, + "tokens_seen": 1914896384 + }, + { + "epoch": 0.15, + "learning_rate": 0.000426949494949495, + "loss": 1.504, + "theoretical_loss": 3.4428968064764813, + "tokens_seen": 1914961920 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042692929292929296, + "loss": 1.3443, + "theoretical_loss": 3.442886898426912, + "tokens_seen": 1915027456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269090909090909, + "loss": 1.2391, + "theoretical_loss": 3.442876990811347, + "tokens_seen": 1915092992 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042688888888888887, + "loss": 1.1644, + "theoretical_loss": 3.4428670836297517, + "tokens_seen": 1915158528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268686868686868, + "loss": 1.4366, + "theoretical_loss": 3.442857176882093, + "tokens_seen": 1915224064 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268484848484849, + "loss": 1.6832, + "theoretical_loss": 3.442847270568336, + "tokens_seen": 1915289600 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042682828282828285, + "loss": 1.41, + "theoretical_loss": 3.4428373646884474, + "tokens_seen": 1915355136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268080808080808, + "loss": 1.37, + "theoretical_loss": 3.4428274592423938, + "tokens_seen": 1915420672 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042678787878787876, + "loss": 1.3526, + "theoretical_loss": 3.4428175542301407, + "tokens_seen": 1915486208 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.41758307814598083, + "objective/train/avg_token_score": 0.019459545612335205, + "objective/train/docs_used": 1083212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3157575130462646, + "objective/train/original_loss": 4.584113121032715, + "objective/train/theoretical_loss": 3.442807649651654, + "objective/train/tokens_used": 274410976, + "objective/train/unlikelihood_loss": 0.04653491452336311, + "theoretical_loss": 3.442807649651654, + "tokens_seen": 1915551744 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042676767676767677, + "loss": 1.4367, + "theoretical_loss": 3.442807649651654, + "tokens_seen": 1915551744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004267474747474748, + "loss": 1.189, + "theoretical_loss": 3.4427977455069008, + "tokens_seen": 1915617280 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042672727272727274, + "loss": 1.3401, + "theoretical_loss": 3.4427878417958464, + "tokens_seen": 1915682816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004267070707070707, + "loss": 1.2258, + "theoretical_loss": 3.442777938518458, + "tokens_seen": 1915748352 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042668686868686865, + "loss": 1.505, + "theoretical_loss": 3.4427680356747006, + "tokens_seen": 1915813888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266666666666667, + "loss": 1.6968, + "theoretical_loss": 3.442758133264541, + "tokens_seen": 1915879424 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266464646464647, + "loss": 1.431, + "theoretical_loss": 3.442748231287946, + "tokens_seen": 1915944960 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042662626262626263, + "loss": 1.4045, + "theoretical_loss": 3.4427383297448806, + "tokens_seen": 1916010496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266060606060606, + "loss": 1.2996, + "theoretical_loss": 3.4427284286353115, + "tokens_seen": 1916076032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265858585858586, + "loss": 1.403, + "theoretical_loss": 3.442718527959205, + "tokens_seen": 1916141568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265656565656566, + "loss": 1.3892, + "theoretical_loss": 3.4427086277165273, + "tokens_seen": 1916207104 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042654545454545457, + "loss": 1.5321, + "theoretical_loss": 3.442698727907244, + "tokens_seen": 1916272640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265252525252525, + "loss": 1.5272, + "theoretical_loss": 3.442688828531322, + "tokens_seen": 1916338176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265050505050505, + "loss": 1.5984, + "theoretical_loss": 3.4426789295887277, + "tokens_seen": 1916403712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264848484848485, + "loss": 1.3496, + "theoretical_loss": 3.4426690310794266, + "tokens_seen": 1916469248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264646464646465, + "loss": 1.3557, + "theoretical_loss": 3.442659133003385, + "tokens_seen": 1916534784 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042644444444444446, + "loss": 1.2853, + "theoretical_loss": 3.4426492353605695, + "tokens_seen": 1916600320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264242424242424, + "loss": 1.5258, + "theoretical_loss": 3.442639338150946, + "tokens_seen": 1916665856 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042640404040404043, + "loss": 1.2848, + "theoretical_loss": 3.4426294413744816, + "tokens_seen": 1916731392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263838383838384, + "loss": 1.4453, + "theoretical_loss": 3.4426195450311416, + "tokens_seen": 1916796928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263636363636364, + "loss": 1.2794, + "theoretical_loss": 3.442609649120892, + "tokens_seen": 1916862464 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042634343434343435, + "loss": 1.2195, + "theoretical_loss": 3.4425997536437, + "tokens_seen": 1916928000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263232323232323, + "loss": 1.6612, + "theoretical_loss": 3.4425898585995314, + "tokens_seen": 1916993536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263030303030303, + "loss": 1.3784, + "theoretical_loss": 3.4425799639883525, + "tokens_seen": 1917059072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262828282828283, + "loss": 1.2362, + "theoretical_loss": 3.442570069810129, + "tokens_seen": 1917124608 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.3271627426147461, + "objective/train/avg_token_score": 0.00935707613825798, + "objective/train/docs_used": 1083753, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1364368200302124, + "objective/train/original_loss": 4.213170051574707, + "objective/train/theoretical_loss": 3.442560176064828, + "objective/train/tokens_used": 276049376, + "objective/train/unlikelihood_loss": 0.05666195973753929, + "theoretical_loss": 3.442560176064828, + "tokens_seen": 1917190144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262626262626263, + "loss": 1.2487, + "theoretical_loss": 3.442560176064828, + "tokens_seen": 1917190144 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042624242424242425, + "loss": 1.3266, + "theoretical_loss": 3.442550282752415, + "tokens_seen": 1917255680 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042622222222222226, + "loss": 1.4237, + "theoretical_loss": 3.4425403898728573, + "tokens_seen": 1917321216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262020202020202, + "loss": 1.3311, + "theoretical_loss": 3.44253049742612, + "tokens_seen": 1917386752 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042618181818181817, + "loss": 1.4461, + "theoretical_loss": 3.44252060541217, + "tokens_seen": 1917452288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261616161616162, + "loss": 1.5083, + "theoretical_loss": 3.4425107138309734, + "tokens_seen": 1917517824 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042614141414141414, + "loss": 1.5493, + "theoretical_loss": 3.4425008226824967, + "tokens_seen": 1917583360 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042612121212121215, + "loss": 1.3401, + "theoretical_loss": 3.442490931966706, + "tokens_seen": 1917648896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261010101010101, + "loss": 1.521, + "theoretical_loss": 3.4424810416835676, + "tokens_seen": 1917714432 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042608080808080806, + "loss": 1.3547, + "theoretical_loss": 3.4424711518330477, + "tokens_seen": 1917779968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260606060606061, + "loss": 1.4561, + "theoretical_loss": 3.442461262415113, + "tokens_seen": 1917845504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260404040404041, + "loss": 1.4914, + "theoretical_loss": 3.442451373429729, + "tokens_seen": 1917911040 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042602020202020204, + "loss": 1.2794, + "theoretical_loss": 3.4424414848768627, + "tokens_seen": 1917976576 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426, + "loss": 1.2327, + "theoretical_loss": 3.4424315967564807, + "tokens_seen": 1918042112 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042597979797979796, + "loss": 1.5526, + "theoretical_loss": 3.4424217090685483, + "tokens_seen": 1918107648 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042595959595959597, + "loss": 1.5492, + "theoretical_loss": 3.4424118218130326, + "tokens_seen": 1918173184 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425939393939394, + "loss": 1.3568, + "theoretical_loss": 3.442401934989899, + "tokens_seen": 1918238720 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042591919191919194, + "loss": 1.3991, + "theoretical_loss": 3.442392048599115, + "tokens_seen": 1918304256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258989898989899, + "loss": 1.3583, + "theoretical_loss": 3.442382162640646, + "tokens_seen": 1918369792 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042587878787878785, + "loss": 1.5928, + "theoretical_loss": 3.442372277114459, + "tokens_seen": 1918435328 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258585858585859, + "loss": 1.1758, + "theoretical_loss": 3.44236239202052, + "tokens_seen": 1918500864 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042583838383838387, + "loss": 1.4569, + "theoretical_loss": 3.4423525073587955, + "tokens_seen": 1918566400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042581818181818183, + "loss": 1.4608, + "theoretical_loss": 3.4423426231292518, + "tokens_seen": 1918631936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257979797979798, + "loss": 1.5675, + "theoretical_loss": 3.442332739331855, + "tokens_seen": 1918697472 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042577777777777774, + "loss": 1.2314, + "theoretical_loss": 3.4423228559665713, + "tokens_seen": 1918763008 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.4659701883792877, + "objective/train/avg_token_score": 0.018756110221147537, + "objective/train/docs_used": 1084888, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6133241653442383, + "objective/train/original_loss": 4.159404277801514, + "objective/train/theoretical_loss": 3.442312973033368, + "objective/train/tokens_used": 277687776, + "objective/train/unlikelihood_loss": 0.05935472249984741, + "theoretical_loss": 3.442312973033368, + "tokens_seen": 1918828544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257575757575758, + "loss": 1.3151, + "theoretical_loss": 3.442312973033368, + "tokens_seen": 1918828544 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042573737373737377, + "loss": 1.4888, + "theoretical_loss": 3.4423030905322105, + "tokens_seen": 1918894080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257171717171717, + "loss": 1.4874, + "theoretical_loss": 3.4422932084630653, + "tokens_seen": 1918959616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256969696969697, + "loss": 1.4814, + "theoretical_loss": 3.442283326825899, + "tokens_seen": 1919025152 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042567676767676764, + "loss": 1.3381, + "theoretical_loss": 3.4422734456206783, + "tokens_seen": 1919090688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256565656565657, + "loss": 1.4287, + "theoretical_loss": 3.4422635648473685, + "tokens_seen": 1919156224 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042563636363636366, + "loss": 1.2571, + "theoretical_loss": 3.442253684505937, + "tokens_seen": 1919221760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256161616161616, + "loss": 1.3749, + "theoretical_loss": 3.4422438045963495, + "tokens_seen": 1919287296 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042559595959595957, + "loss": 1.6466, + "theoretical_loss": 3.4422339251185727, + "tokens_seen": 1919352832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255757575757576, + "loss": 1.3934, + "theoretical_loss": 3.4422240460725737, + "tokens_seen": 1919418368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255555555555556, + "loss": 1.2283, + "theoretical_loss": 3.4422141674583173, + "tokens_seen": 1919483904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042553535353535355, + "loss": 1.3776, + "theoretical_loss": 3.4422042892757716, + "tokens_seen": 1919549440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255151515151515, + "loss": 1.4192, + "theoretical_loss": 3.4421944115249015, + "tokens_seen": 1919614976 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042549494949494946, + "loss": 1.4077, + "theoretical_loss": 3.442184534205674, + "tokens_seen": 1919680512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254747474747475, + "loss": 1.3611, + "theoretical_loss": 3.4421746573180556, + "tokens_seen": 1919746048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254545454545455, + "loss": 1.3919, + "theoretical_loss": 3.4421647808620133, + "tokens_seen": 1919811584 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042543434343434344, + "loss": 1.6059, + "theoretical_loss": 3.442154904837512, + "tokens_seen": 1919877120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254141414141414, + "loss": 1.4137, + "theoretical_loss": 3.4421450292445197, + "tokens_seen": 1919942656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253939393939394, + "loss": 1.4409, + "theoretical_loss": 3.442135154083002, + "tokens_seen": 1920008192 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042537373737373737, + "loss": 1.2995, + "theoretical_loss": 3.4421252793529247, + "tokens_seen": 1920073728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253535353535354, + "loss": 1.457, + "theoretical_loss": 3.4421154050542553, + "tokens_seen": 1920139264 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042533333333333334, + "loss": 1.3346, + "theoretical_loss": 3.4421055311869604, + "tokens_seen": 1920204800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253131313131313, + "loss": 1.4856, + "theoretical_loss": 3.442095657751006, + "tokens_seen": 1920270336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252929292929293, + "loss": 1.4931, + "theoretical_loss": 3.4420857847463573, + "tokens_seen": 1920335872 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042527272727272726, + "loss": 1.4235, + "theoretical_loss": 3.442075912172983, + "tokens_seen": 1920401408 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.2597140669822693, + "objective/train/avg_token_score": 0.00819999910891056, + "objective/train/docs_used": 1085471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8639290928840637, + "objective/train/original_loss": 3.9054853916168213, + "objective/train/theoretical_loss": 3.4420660400308476, + "objective/train/tokens_used": 279326176, + "objective/train/unlikelihood_loss": 0.08530934154987335, + "theoretical_loss": 3.4420660400308476, + "tokens_seen": 1920466944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042525252525252527, + "loss": 1.29, + "theoretical_loss": 3.4420660400308476, + "tokens_seen": 1920466944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042523232323232323, + "loss": 1.4392, + "theoretical_loss": 3.442056168319919, + "tokens_seen": 1920532480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042521212121212124, + "loss": 1.5414, + "theoretical_loss": 3.442046297040163, + "tokens_seen": 1920598016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251919191919192, + "loss": 1.5928, + "theoretical_loss": 3.442036426191546, + "tokens_seen": 1920663552 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042517171717171715, + "loss": 1.5551, + "theoretical_loss": 3.442026555774034, + "tokens_seen": 1920729088 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042515151515151516, + "loss": 1.4526, + "theoretical_loss": 3.442016685787595, + "tokens_seen": 1920794624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251313131313131, + "loss": 1.3303, + "theoretical_loss": 3.442006816232194, + "tokens_seen": 1920860160 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042511111111111113, + "loss": 1.4716, + "theoretical_loss": 3.441996947107798, + "tokens_seen": 1920925696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250909090909091, + "loss": 1.7319, + "theoretical_loss": 3.441987078414373, + "tokens_seen": 1920991232 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042507070707070705, + "loss": 1.298, + "theoretical_loss": 3.4419772101518866, + "tokens_seen": 1921056768 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042505050505050506, + "loss": 1.4499, + "theoretical_loss": 3.4419673423203037, + "tokens_seen": 1921122304 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042503030303030307, + "loss": 1.4579, + "theoretical_loss": 3.4419574749195925, + "tokens_seen": 1921187840 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425010101010101, + "loss": 1.3127, + "theoretical_loss": 3.441947607949718, + "tokens_seen": 1921253376 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424989898989899, + "loss": 1.3889, + "theoretical_loss": 3.441937741410648, + "tokens_seen": 1921318912 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424969696969697, + "loss": 1.3067, + "theoretical_loss": 3.441927875302348, + "tokens_seen": 1921384448 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042494949494949495, + "loss": 1.2578, + "theoretical_loss": 3.441918009624785, + "tokens_seen": 1921449984 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042492929292929296, + "loss": 1.2618, + "theoretical_loss": 3.4419081443779254, + "tokens_seen": 1921515520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249090909090909, + "loss": 1.5736, + "theoretical_loss": 3.4418982795617357, + "tokens_seen": 1921581056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248888888888889, + "loss": 1.4578, + "theoretical_loss": 3.441888415176182, + "tokens_seen": 1921646592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248686868686869, + "loss": 1.3573, + "theoretical_loss": 3.4418785512212313, + "tokens_seen": 1921712128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248484848484849, + "loss": 1.3088, + "theoretical_loss": 3.44186868769685, + "tokens_seen": 1921777664 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042482828282828285, + "loss": 1.2699, + "theoretical_loss": 3.441858824603005, + "tokens_seen": 1921843200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248080808080808, + "loss": 1.4885, + "theoretical_loss": 3.4418489619396624, + "tokens_seen": 1921908736 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042478787878787877, + "loss": 1.4237, + "theoretical_loss": 3.4418390997067885, + "tokens_seen": 1921974272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247676767676768, + "loss": 1.4151, + "theoretical_loss": 3.44182923790435, + "tokens_seen": 1922039808 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.37664955854415894, + "objective/train/avg_token_score": 0.027382804080843925, + "objective/train/docs_used": 1086847, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2897065877914429, + "objective/train/original_loss": 4.506113529205322, + "objective/train/theoretical_loss": 3.4418193765323135, + "objective/train/tokens_used": 280964576, + "objective/train/unlikelihood_loss": 0.06149740517139435, + "theoretical_loss": 3.4418193765323135, + "tokens_seen": 1922105344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247474747474748, + "loss": 1.3919, + "theoretical_loss": 3.4418193765323135, + "tokens_seen": 1922105344 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042472727272727275, + "loss": 1.1067, + "theoretical_loss": 3.4418095155906463, + "tokens_seen": 1922170880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247070707070707, + "loss": 1.3229, + "theoretical_loss": 3.441799655079314, + "tokens_seen": 1922236416 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042468686868686866, + "loss": 1.283, + "theoretical_loss": 3.441789794998283, + "tokens_seen": 1922301952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246666666666667, + "loss": 1.3661, + "theoretical_loss": 3.4417799353475202, + "tokens_seen": 1922367488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246464646464647, + "loss": 1.3491, + "theoretical_loss": 3.4417700761269927, + "tokens_seen": 1922433024 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042462626262626264, + "loss": 1.4425, + "theoretical_loss": 3.4417602173366664, + "tokens_seen": 1922498560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246060606060606, + "loss": 1.5416, + "theoretical_loss": 3.441750358976508, + "tokens_seen": 1922564096 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042458585858585855, + "loss": 1.2915, + "theoretical_loss": 3.4417405010464837, + "tokens_seen": 1922629632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245656565656566, + "loss": 1.7483, + "theoretical_loss": 3.441730643546561, + "tokens_seen": 1922695168 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245454545454546, + "loss": 1.4832, + "theoretical_loss": 3.441720786476706, + "tokens_seen": 1922760704 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042452525252525253, + "loss": 1.5824, + "theoretical_loss": 3.441710929836885, + "tokens_seen": 1922826240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245050505050505, + "loss": 1.5308, + "theoretical_loss": 3.4417010736270646, + "tokens_seen": 1922891776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244848484848485, + "loss": 1.2997, + "theoretical_loss": 3.441691217847212, + "tokens_seen": 1922957312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244646464646465, + "loss": 1.4648, + "theoretical_loss": 3.4416813624972935, + "tokens_seen": 1923022848 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042444444444444447, + "loss": 1.1912, + "theoretical_loss": 3.441671507577275, + "tokens_seen": 1923088384 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244242424242424, + "loss": 1.3946, + "theoretical_loss": 3.441661653087124, + "tokens_seen": 1923153920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244040404040404, + "loss": 1.4577, + "theoretical_loss": 3.4416517990268067, + "tokens_seen": 1923219456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243838383838384, + "loss": 1.2794, + "theoretical_loss": 3.44164194539629, + "tokens_seen": 1923284992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243636363636364, + "loss": 1.4351, + "theoretical_loss": 3.4416320921955403, + "tokens_seen": 1923350528 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042434343434343436, + "loss": 1.4165, + "theoretical_loss": 3.4416222394245235, + "tokens_seen": 1923416064 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243232323232323, + "loss": 1.4822, + "theoretical_loss": 3.441612387083208, + "tokens_seen": 1923481600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042430303030303033, + "loss": 1.2961, + "theoretical_loss": 3.4416025351715587, + "tokens_seen": 1923547136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242828282828283, + "loss": 1.5846, + "theoretical_loss": 3.441592683689543, + "tokens_seen": 1923612672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242626262626263, + "loss": 1.5942, + "theoretical_loss": 3.4415828326371276, + "tokens_seen": 1923678208 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.353128045797348, + "objective/train/avg_token_score": 0.019363651052117348, + "objective/train/docs_used": 1087440, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.210463047027588, + "objective/train/original_loss": 4.447400093078613, + "objective/train/theoretical_loss": 3.4415729820142786, + "objective/train/tokens_used": 282602976, + "objective/train/unlikelihood_loss": 0.08732765913009644, + "theoretical_loss": 3.4415729820142786, + "tokens_seen": 1923743744 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042424242424242425, + "loss": 1.3893, + "theoretical_loss": 3.4415729820142786, + "tokens_seen": 1923743744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242222222222222, + "loss": 1.3323, + "theoretical_loss": 3.441563131820963, + "tokens_seen": 1923809280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242020202020202, + "loss": 1.3105, + "theoretical_loss": 3.4415532820571473, + "tokens_seen": 1923874816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241818181818182, + "loss": 1.3959, + "theoretical_loss": 3.4415434327227983, + "tokens_seen": 1923940352 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241616161616162, + "loss": 1.5941, + "theoretical_loss": 3.441533583817883, + "tokens_seen": 1924005888 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042414141414141415, + "loss": 1.3544, + "theoretical_loss": 3.4415237353423676, + "tokens_seen": 1924071424 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042412121212121216, + "loss": 1.3194, + "theoretical_loss": 3.4415138872962183, + "tokens_seen": 1924136960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241010101010101, + "loss": 1.4216, + "theoretical_loss": 3.4415040396794025, + "tokens_seen": 1924202496 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042408080808080807, + "loss": 1.4374, + "theoretical_loss": 3.441494192491887, + "tokens_seen": 1924268032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240606060606061, + "loss": 1.6304, + "theoretical_loss": 3.4414843457336373, + "tokens_seen": 1924333568 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042404040404040404, + "loss": 1.2735, + "theoretical_loss": 3.441474499404621, + "tokens_seen": 1924399104 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042402020202020205, + "loss": 1.325, + "theoretical_loss": 3.441464653504805, + "tokens_seen": 1924464640 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424, + "loss": 1.6185, + "theoretical_loss": 3.4414548080341554, + "tokens_seen": 1924530176 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042397979797979796, + "loss": 1.3112, + "theoretical_loss": 3.441444962992639, + "tokens_seen": 1924595712 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423959595959596, + "loss": 1.6121, + "theoretical_loss": 3.441435118380223, + "tokens_seen": 1924661248 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042393939393939393, + "loss": 1.4117, + "theoretical_loss": 3.4414252741968734, + "tokens_seen": 1924726784 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042391919191919194, + "loss": 1.3539, + "theoretical_loss": 3.4414154304425573, + "tokens_seen": 1924792320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238989898989899, + "loss": 1.2487, + "theoretical_loss": 3.4414055871172406, + "tokens_seen": 1924857856 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042387878787878786, + "loss": 1.4242, + "theoretical_loss": 3.441395744220891, + "tokens_seen": 1924923392 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042385858585858587, + "loss": 1.4151, + "theoretical_loss": 3.441385901753475, + "tokens_seen": 1924988928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238383838383839, + "loss": 1.4439, + "theoretical_loss": 3.441376059714959, + "tokens_seen": 1925054464 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042381818181818183, + "loss": 1.5184, + "theoretical_loss": 3.44136621810531, + "tokens_seen": 1925120000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237979797979798, + "loss": 1.5182, + "theoretical_loss": 3.4413563769244946, + "tokens_seen": 1925185536 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042377777777777775, + "loss": 1.4028, + "theoretical_loss": 3.441346536172479, + "tokens_seen": 1925251072 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042375757575757576, + "loss": 1.3353, + "theoretical_loss": 3.441336695849231, + "tokens_seen": 1925316608 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.48949170112609863, + "objective/train/avg_token_score": 0.014318176545202732, + "objective/train/docs_used": 1088918, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6369242668151855, + "objective/train/original_loss": 4.034110069274902, + "objective/train/theoretical_loss": 3.441326855954716, + "objective/train/tokens_used": 284241376, + "objective/train/unlikelihood_loss": 0.05818219855427742, + "theoretical_loss": 3.441326855954716, + "tokens_seen": 1925382144 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042373737373737377, + "loss": 1.5529, + "theoretical_loss": 3.441326855954716, + "tokens_seen": 1925382144 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042371717171717173, + "loss": 1.3521, + "theoretical_loss": 3.4413170164889024, + "tokens_seen": 1925447680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236969696969697, + "loss": 1.267, + "theoretical_loss": 3.441307177451755, + "tokens_seen": 1925513216 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042367676767676764, + "loss": 1.6063, + "theoretical_loss": 3.4412973388432424, + "tokens_seen": 1925578752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236565656565657, + "loss": 1.4146, + "theoretical_loss": 3.44128750066333, + "tokens_seen": 1925644288 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042363636363636366, + "loss": 1.6882, + "theoretical_loss": 3.4412776629119852, + "tokens_seen": 1925709824 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236161616161616, + "loss": 1.3858, + "theoretical_loss": 3.441267825589174, + "tokens_seen": 1925775360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004235959595959596, + "loss": 1.4042, + "theoretical_loss": 3.4412579886948644, + "tokens_seen": 1925840896 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042357575757575753, + "loss": 1.3035, + "theoretical_loss": 3.441248152229022, + "tokens_seen": 1925906432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004235555555555556, + "loss": 1.4533, + "theoretical_loss": 3.4412383161916145, + "tokens_seen": 1925971968 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042353535353535356, + "loss": 1.743, + "theoretical_loss": 3.4412284805826077, + "tokens_seen": 1926037504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004235151515151515, + "loss": 1.2599, + "theoretical_loss": 3.441218645401969, + "tokens_seen": 1926103040 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042349494949494947, + "loss": 1.4499, + "theoretical_loss": 3.441208810649665, + "tokens_seen": 1926168576 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234747474747475, + "loss": 1.5833, + "theoretical_loss": 3.4411989763256625, + "tokens_seen": 1926234112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234545454545455, + "loss": 1.3539, + "theoretical_loss": 3.441189142429928, + "tokens_seen": 1926299648 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042343434343434345, + "loss": 1.4448, + "theoretical_loss": 3.441179308962429, + "tokens_seen": 1926365184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234141414141414, + "loss": 1.485, + "theoretical_loss": 3.4411694759231315, + "tokens_seen": 1926430720 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042339393939393936, + "loss": 1.6433, + "theoretical_loss": 3.4411596433120026, + "tokens_seen": 1926496256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233737373737374, + "loss": 1.311, + "theoretical_loss": 3.4411498111290095, + "tokens_seen": 1926561792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233535353535354, + "loss": 1.408, + "theoretical_loss": 3.4411399793741184, + "tokens_seen": 1926627328 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042333333333333334, + "loss": 1.3597, + "theoretical_loss": 3.441130148047296, + "tokens_seen": 1926692864 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233131313131313, + "loss": 1.4085, + "theoretical_loss": 3.44112031714851, + "tokens_seen": 1926758400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232929292929293, + "loss": 1.2997, + "theoretical_loss": 3.441110486677726, + "tokens_seen": 1926823936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232727272727273, + "loss": 1.3375, + "theoretical_loss": 3.4411006566349114, + "tokens_seen": 1926889472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232525252525253, + "loss": 1.3776, + "theoretical_loss": 3.441090827020033, + "tokens_seen": 1926955008 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.4183773100376129, + "objective/train/avg_token_score": 0.027611570432782173, + "objective/train/docs_used": 1089550, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.40208101272583, + "objective/train/original_loss": 3.778869867324829, + "objective/train/theoretical_loss": 3.4410809978330583, + "objective/train/tokens_used": 285879776, + "objective/train/unlikelihood_loss": 0.07978764176368713, + "theoretical_loss": 3.4410809978330583, + "tokens_seen": 1927020544 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042323232323232323, + "loss": 1.1572, + "theoretical_loss": 3.4410809978330583, + "tokens_seen": 1927020544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232121212121212, + "loss": 1.3984, + "theoretical_loss": 3.441071169073953, + "tokens_seen": 1927086080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231919191919192, + "loss": 1.5348, + "theoretical_loss": 3.441061340742684, + "tokens_seen": 1927151616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231717171717172, + "loss": 1.6712, + "theoretical_loss": 3.4410515128392194, + "tokens_seen": 1927217152 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042315151515151517, + "loss": 1.43, + "theoretical_loss": 3.4410416853635244, + "tokens_seen": 1927282688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231313131313131, + "loss": 1.5115, + "theoretical_loss": 3.441031858315567, + "tokens_seen": 1927348224 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042311111111111114, + "loss": 1.5486, + "theoretical_loss": 3.4410220316953133, + "tokens_seen": 1927413760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230909090909091, + "loss": 1.408, + "theoretical_loss": 3.441012205502731, + "tokens_seen": 1927479296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230707070707071, + "loss": 1.483, + "theoretical_loss": 3.441002379737786, + "tokens_seen": 1927544832 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042305050505050506, + "loss": 1.5122, + "theoretical_loss": 3.4409925544004456, + "tokens_seen": 1927610368 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423030303030303, + "loss": 1.4116, + "theoretical_loss": 3.4409827294906767, + "tokens_seen": 1927675904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042301010101010103, + "loss": 1.561, + "theoretical_loss": 3.4409729050084463, + "tokens_seen": 1927741440 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422989898989899, + "loss": 1.5038, + "theoretical_loss": 3.440963080953721, + "tokens_seen": 1927806976 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422969696969697, + "loss": 1.3527, + "theoretical_loss": 3.440953257326467, + "tokens_seen": 1927872512 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042294949494949496, + "loss": 1.3933, + "theoretical_loss": 3.4409434341266527, + "tokens_seen": 1927938048 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042292929292929297, + "loss": 1.6347, + "theoretical_loss": 3.440933611354244, + "tokens_seen": 1928003584 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229090909090909, + "loss": 1.3893, + "theoretical_loss": 3.4409237890092075, + "tokens_seen": 1928069120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228888888888889, + "loss": 1.0593, + "theoretical_loss": 3.440913967091511, + "tokens_seen": 1928134656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228686868686869, + "loss": 1.2617, + "theoretical_loss": 3.4409041456011202, + "tokens_seen": 1928200192 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042284848484848485, + "loss": 1.2376, + "theoretical_loss": 3.4408943245380033, + "tokens_seen": 1928265728 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042282828282828286, + "loss": 1.4189, + "theoretical_loss": 3.4408845039021263, + "tokens_seen": 1928331264 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228080808080808, + "loss": 1.3413, + "theoretical_loss": 3.4408746836934565, + "tokens_seen": 1928396800 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042278787878787877, + "loss": 1.3963, + "theoretical_loss": 3.4408648639119606, + "tokens_seen": 1928462336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227676767676768, + "loss": 1.2697, + "theoretical_loss": 3.4408550445576056, + "tokens_seen": 1928527872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227474747474748, + "loss": 1.7067, + "theoretical_loss": 3.4408452256303583, + "tokens_seen": 1928593408 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.30810117721557617, + "objective/train/avg_token_score": 0.016539035364985466, + "objective/train/docs_used": 1091004, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9405573606491089, + "objective/train/original_loss": 4.469072341918945, + "objective/train/theoretical_loss": 3.4408354071301854, + "objective/train/tokens_used": 287518176, + "objective/train/unlikelihood_loss": 0.0698821097612381, + "theoretical_loss": 3.4408354071301854, + "tokens_seen": 1928658944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042272727272727275, + "loss": 1.2846, + "theoretical_loss": 3.4408354071301854, + "tokens_seen": 1928658944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227070707070707, + "loss": 1.4077, + "theoretical_loss": 3.4408255890570545, + "tokens_seen": 1928724480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042268686868686867, + "loss": 1.5451, + "theoretical_loss": 3.440815771410932, + "tokens_seen": 1928790016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226666666666667, + "loss": 1.4673, + "theoretical_loss": 3.440805954191785, + "tokens_seen": 1928855552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226464646464647, + "loss": 1.5765, + "theoretical_loss": 3.4407961373995803, + "tokens_seen": 1928921088 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042262626262626264, + "loss": 1.4946, + "theoretical_loss": 3.4407863210342846, + "tokens_seen": 1928986624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226060606060606, + "loss": 1.6541, + "theoretical_loss": 3.440776505095865, + "tokens_seen": 1929052160 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042258585858585856, + "loss": 1.4803, + "theoretical_loss": 3.440766689584289, + "tokens_seen": 1929117696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225656565656566, + "loss": 1.5839, + "theoretical_loss": 3.4407568744995234, + "tokens_seen": 1929183232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225454545454546, + "loss": 1.4868, + "theoretical_loss": 3.4407470598415344, + "tokens_seen": 1929248768 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042252525252525254, + "loss": 1.3395, + "theoretical_loss": 3.440737245610289, + "tokens_seen": 1929314304 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225050505050505, + "loss": 1.437, + "theoretical_loss": 3.4407274318057555, + "tokens_seen": 1929379840 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042248484848484845, + "loss": 1.5503, + "theoretical_loss": 3.440717618427899, + "tokens_seen": 1929445376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224646464646465, + "loss": 1.3331, + "theoretical_loss": 3.4407078054766878, + "tokens_seen": 1929510912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224444444444445, + "loss": 1.4377, + "theoretical_loss": 3.440697992952088, + "tokens_seen": 1929576448 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042242424242424243, + "loss": 1.5041, + "theoretical_loss": 3.4406881808540675, + "tokens_seen": 1929641984 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224040404040404, + "loss": 1.4972, + "theoretical_loss": 3.440678369182592, + "tokens_seen": 1929707520 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042238383838383834, + "loss": 1.4901, + "theoretical_loss": 3.44066855793763, + "tokens_seen": 1929773056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223636363636364, + "loss": 1.3971, + "theoretical_loss": 3.440658747119147, + "tokens_seen": 1929838592 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042234343434343437, + "loss": 1.6506, + "theoretical_loss": 3.440648936727111, + "tokens_seen": 1929904128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223232323232323, + "loss": 1.6115, + "theoretical_loss": 3.4406391267614884, + "tokens_seen": 1929969664 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223030303030303, + "loss": 1.4711, + "theoretical_loss": 3.4406293172222466, + "tokens_seen": 1930035200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222828282828283, + "loss": 1.5098, + "theoretical_loss": 3.4406195081093527, + "tokens_seen": 1930100736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222626262626263, + "loss": 1.3378, + "theoretical_loss": 3.4406096994227733, + "tokens_seen": 1930166272 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042224242424242426, + "loss": 1.5783, + "theoretical_loss": 3.440599891162475, + "tokens_seen": 1930231808 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.6237170100212097, + "objective/train/avg_token_score": 0.006665772292762995, + "objective/train/docs_used": 1091522, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7739847898483276, + "objective/train/original_loss": 3.4937384128570557, + "objective/train/theoretical_loss": 3.4405900833284258, + "objective/train/tokens_used": 289156576, + "objective/train/unlikelihood_loss": 0.047466766089200974, + "theoretical_loss": 3.4405900833284258, + "tokens_seen": 1930297344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222222222222222, + "loss": 1.4705, + "theoretical_loss": 3.4405900833284258, + "tokens_seen": 1930297344 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042220202020202017, + "loss": 1.3696, + "theoretical_loss": 3.440580275920592, + "tokens_seen": 1930362880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221818181818182, + "loss": 1.3453, + "theoretical_loss": 3.440570468938941, + "tokens_seen": 1930428416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221616161616162, + "loss": 1.6253, + "theoretical_loss": 3.4405606623834393, + "tokens_seen": 1930493952 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042214141414141415, + "loss": 1.5188, + "theoretical_loss": 3.4405508562540543, + "tokens_seen": 1930559488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221212121212121, + "loss": 1.4635, + "theoretical_loss": 3.4405410505507534, + "tokens_seen": 1930625024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221010101010101, + "loss": 1.3429, + "theoretical_loss": 3.440531245273503, + "tokens_seen": 1930690560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220808080808081, + "loss": 1.319, + "theoretical_loss": 3.4405214404222706, + "tokens_seen": 1930756096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220606060606061, + "loss": 1.2327, + "theoretical_loss": 3.4405116359970225, + "tokens_seen": 1930821632 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042204040404040404, + "loss": 1.5396, + "theoretical_loss": 3.4405018319977265, + "tokens_seen": 1930887168 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422020202020202, + "loss": 1.5807, + "theoretical_loss": 3.440492028424349, + "tokens_seen": 1930952704 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422, + "loss": 1.3657, + "theoretical_loss": 3.440482225276858, + "tokens_seen": 1931018240 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042197979797979797, + "loss": 1.2815, + "theoretical_loss": 3.440472422555219, + "tokens_seen": 1931083776 + }, + { + "epoch": 0.16, + "learning_rate": 0.000421959595959596, + "loss": 1.2827, + "theoretical_loss": 3.4404626202594004, + "tokens_seen": 1931149312 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042193939393939394, + "loss": 1.4397, + "theoretical_loss": 3.4404528183893692, + "tokens_seen": 1931214848 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042191919191919195, + "loss": 1.7767, + "theoretical_loss": 3.4404430169450917, + "tokens_seen": 1931280384 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218989898989899, + "loss": 1.6945, + "theoretical_loss": 3.4404332159265354, + "tokens_seen": 1931345920 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042187878787878786, + "loss": 1.3724, + "theoretical_loss": 3.440423415333667, + "tokens_seen": 1931411456 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042185858585858587, + "loss": 1.2764, + "theoretical_loss": 3.440413615166454, + "tokens_seen": 1931476992 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042183838383838383, + "loss": 1.4126, + "theoretical_loss": 3.440403815424864, + "tokens_seen": 1931542528 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042181818181818184, + "loss": 1.3462, + "theoretical_loss": 3.440394016108863, + "tokens_seen": 1931608064 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217979797979798, + "loss": 1.3443, + "theoretical_loss": 3.4403842172184183, + "tokens_seen": 1931673600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042177777777777775, + "loss": 1.5187, + "theoretical_loss": 3.440374418753497, + "tokens_seen": 1931739136 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042175757575757577, + "loss": 1.4158, + "theoretical_loss": 3.4403646207140666, + "tokens_seen": 1931804672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217373737373738, + "loss": 1.5499, + "theoretical_loss": 3.440354823100094, + "tokens_seen": 1931870208 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.4935239553451538, + "objective/train/avg_token_score": 0.006382493767887354, + "objective/train/docs_used": 1092471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.711350440979004, + "objective/train/original_loss": 3.7253687381744385, + "objective/train/theoretical_loss": 3.4403450259115465, + "objective/train/tokens_used": 290794976, + "objective/train/unlikelihood_loss": 0.1075059249997139, + "theoretical_loss": 3.4403450259115465, + "tokens_seen": 1931935744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042171717171717173, + "loss": 1.5274, + "theoretical_loss": 3.4403450259115465, + "tokens_seen": 1931935744 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216969696969697, + "loss": 1.6173, + "theoretical_loss": 3.4403352291483906, + "tokens_seen": 1932001280 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042167676767676765, + "loss": 1.55, + "theoretical_loss": 3.440325432810594, + "tokens_seen": 1932066816 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042165656565656566, + "loss": 1.4703, + "theoretical_loss": 3.4403156368981236, + "tokens_seen": 1932132352 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042163636363636367, + "loss": 1.3505, + "theoretical_loss": 3.4403058414109458, + "tokens_seen": 1932197888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216161616161616, + "loss": 1.3519, + "theoretical_loss": 3.4402960463490286, + "tokens_seen": 1932263424 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215959595959596, + "loss": 1.5252, + "theoretical_loss": 3.4402862517123394, + "tokens_seen": 1932328960 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042157575757575754, + "loss": 1.5055, + "theoretical_loss": 3.440276457500844, + "tokens_seen": 1932394496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215555555555556, + "loss": 1.6192, + "theoretical_loss": 3.440266663714511, + "tokens_seen": 1932460032 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042153535353535356, + "loss": 1.56, + "theoretical_loss": 3.440256870353307, + "tokens_seen": 1932525568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215151515151515, + "loss": 1.5159, + "theoretical_loss": 3.4402470774171987, + "tokens_seen": 1932591104 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214949494949495, + "loss": 1.418, + "theoretical_loss": 3.4402372849061535, + "tokens_seen": 1932656640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214747474747475, + "loss": 1.3176, + "theoretical_loss": 3.4402274928201386, + "tokens_seen": 1932722176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214545454545455, + "loss": 1.3761, + "theoretical_loss": 3.4402177011591206, + "tokens_seen": 1932787712 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042143434343434345, + "loss": 1.3156, + "theoretical_loss": 3.4402079099230676, + "tokens_seen": 1932853248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214141414141414, + "loss": 1.4477, + "theoretical_loss": 3.4401981191119466, + "tokens_seen": 1932918784 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042139393939393937, + "loss": 1.3894, + "theoretical_loss": 3.440188328725724, + "tokens_seen": 1932984320 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042137373737373743, + "loss": 1.4207, + "theoretical_loss": 3.4401785387643677, + "tokens_seen": 1933049856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213535353535354, + "loss": 1.3732, + "theoretical_loss": 3.440168749227844, + "tokens_seen": 1933115392 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042133333333333335, + "loss": 1.4545, + "theoretical_loss": 3.440158960116121, + "tokens_seen": 1933180928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213131313131313, + "loss": 1.4779, + "theoretical_loss": 3.4401491714291654, + "tokens_seen": 1933246464 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042129292929292926, + "loss": 1.3513, + "theoretical_loss": 3.4401393831669447, + "tokens_seen": 1933312000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212727272727273, + "loss": 1.4636, + "theoretical_loss": 3.440129595329425, + "tokens_seen": 1933377536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212525252525253, + "loss": 1.3884, + "theoretical_loss": 3.440119807916575, + "tokens_seen": 1933443072 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042123232323232324, + "loss": 1.3359, + "theoretical_loss": 3.440110020928361, + "tokens_seen": 1933508608 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.2790200412273407, + "objective/train/avg_token_score": 0.020743899047374725, + "objective/train/docs_used": 1092955, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.856033444404602, + "objective/train/original_loss": 4.6658196449279785, + "objective/train/theoretical_loss": 3.440100234364751, + "objective/train/tokens_used": 292433376, + "objective/train/unlikelihood_loss": 0.06594061106443405, + "theoretical_loss": 3.440100234364751, + "tokens_seen": 1933574144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212121212121212, + "loss": 1.2959, + "theoretical_loss": 3.440100234364751, + "tokens_seen": 1933574144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211919191919192, + "loss": 1.3046, + "theoretical_loss": 3.4400904482257104, + "tokens_seen": 1933639680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211717171717172, + "loss": 1.4219, + "theoretical_loss": 3.4400806625112086, + "tokens_seen": 1933705216 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211515151515152, + "loss": 1.3372, + "theoretical_loss": 3.440070877221211, + "tokens_seen": 1933770752 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042113131313131313, + "loss": 1.5524, + "theoretical_loss": 3.440061092355686, + "tokens_seen": 1933836288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211111111111111, + "loss": 1.4611, + "theoretical_loss": 3.4400513079146, + "tokens_seen": 1933901824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210909090909091, + "loss": 1.4109, + "theoretical_loss": 3.440041523897921, + "tokens_seen": 1933967360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210707070707071, + "loss": 1.488, + "theoretical_loss": 3.4400317403056153, + "tokens_seen": 1934032896 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042105050505050507, + "loss": 1.3147, + "theoretical_loss": 3.4400219571376507, + "tokens_seen": 1934098432 + }, + { + "epoch": 0.17, + "learning_rate": 0.000421030303030303, + "loss": 1.5583, + "theoretical_loss": 3.4400121743939946, + "tokens_seen": 1934163968 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042101010101010104, + "loss": 1.6352, + "theoretical_loss": 3.4400023920746134, + "tokens_seen": 1934229504 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420989898989899, + "loss": 1.4111, + "theoretical_loss": 3.439992610179475, + "tokens_seen": 1934295040 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420969696969697, + "loss": 1.7752, + "theoretical_loss": 3.4399828287085468, + "tokens_seen": 1934360576 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042094949494949496, + "loss": 1.4621, + "theoretical_loss": 3.4399730476617956, + "tokens_seen": 1934426112 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209292929292929, + "loss": 1.3782, + "theoretical_loss": 3.4399632670391886, + "tokens_seen": 1934491648 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042090909090909093, + "loss": 1.3861, + "theoretical_loss": 3.439953486840693, + "tokens_seen": 1934557184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208888888888889, + "loss": 1.51, + "theoretical_loss": 3.4399437070662766, + "tokens_seen": 1934622720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208686868686869, + "loss": 1.2562, + "theoretical_loss": 3.439933927715906, + "tokens_seen": 1934688256 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042084848484848485, + "loss": 1.3318, + "theoretical_loss": 3.439924148789549, + "tokens_seen": 1934753792 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042082828282828286, + "loss": 1.4035, + "theoretical_loss": 3.4399143702871724, + "tokens_seen": 1934819328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208080808080808, + "loss": 1.5151, + "theoretical_loss": 3.4399045922087437, + "tokens_seen": 1934884864 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207878787878788, + "loss": 1.4422, + "theoretical_loss": 3.4398948145542296, + "tokens_seen": 1934950400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207676767676768, + "loss": 1.4189, + "theoretical_loss": 3.4398850373235987, + "tokens_seen": 1935015936 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042074747474747475, + "loss": 1.4982, + "theoretical_loss": 3.439875260516817, + "tokens_seen": 1935081472 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042072727272727276, + "loss": 1.2191, + "theoretical_loss": 3.439865484133852, + "tokens_seen": 1935147008 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.492118775844574, + "objective/train/avg_token_score": 0.013941711746156216, + "objective/train/docs_used": 1093966, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5089529752731323, + "objective/train/original_loss": 4.055489540100098, + "objective/train/theoretical_loss": 3.4398557081746715, + "objective/train/tokens_used": 294071776, + "objective/train/unlikelihood_loss": 0.046310193836688995, + "theoretical_loss": 3.4398557081746715, + "tokens_seen": 1935212544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207070707070707, + "loss": 1.4559, + "theoretical_loss": 3.4398557081746715, + "tokens_seen": 1935212544 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042068686868686867, + "loss": 1.6231, + "theoretical_loss": 3.439845932639242, + "tokens_seen": 1935278080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004206666666666667, + "loss": 1.3089, + "theoretical_loss": 3.4398361575275316, + "tokens_seen": 1935343616 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042064646464646464, + "loss": 1.6297, + "theoretical_loss": 3.439826382839507, + "tokens_seen": 1935409152 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042062626262626265, + "loss": 1.5352, + "theoretical_loss": 3.4398166085751356, + "tokens_seen": 1935474688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004206060606060606, + "loss": 1.6264, + "theoretical_loss": 3.4398068347343855, + "tokens_seen": 1935540224 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042058585858585856, + "loss": 1.6027, + "theoretical_loss": 3.439797061317223, + "tokens_seen": 1935605760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205656565656566, + "loss": 1.5475, + "theoretical_loss": 3.439787288323615, + "tokens_seen": 1935671296 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205454545454546, + "loss": 1.3246, + "theoretical_loss": 3.43977751575353, + "tokens_seen": 1935736832 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042052525252525254, + "loss": 1.4094, + "theoretical_loss": 3.439767743606935, + "tokens_seen": 1935802368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205050505050505, + "loss": 1.2892, + "theoretical_loss": 3.439757971883797, + "tokens_seen": 1935867904 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042048484848484846, + "loss": 1.4243, + "theoretical_loss": 3.4397482005840834, + "tokens_seen": 1935933440 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042046464646464647, + "loss": 1.6329, + "theoretical_loss": 3.439738429707762, + "tokens_seen": 1935998976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204444444444445, + "loss": 1.4389, + "theoretical_loss": 3.439728659254799, + "tokens_seen": 1936064512 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042042424242424244, + "loss": 1.3894, + "theoretical_loss": 3.439718889225163, + "tokens_seen": 1936130048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204040404040404, + "loss": 1.3502, + "theoretical_loss": 3.4397091196188203, + "tokens_seen": 1936195584 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042038383838383835, + "loss": 1.6292, + "theoretical_loss": 3.4396993504357387, + "tokens_seen": 1936261120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004203636363636364, + "loss": 1.271, + "theoretical_loss": 3.4396895816758857, + "tokens_seen": 1936326656 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042034343434343437, + "loss": 1.6302, + "theoretical_loss": 3.439679813339229, + "tokens_seen": 1936392192 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042032323232323233, + "loss": 1.3143, + "theoretical_loss": 3.439670045425734, + "tokens_seen": 1936457728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004203030303030303, + "loss": 1.597, + "theoretical_loss": 3.4396602779353707, + "tokens_seen": 1936523264 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042028282828282824, + "loss": 1.5688, + "theoretical_loss": 3.4396505108681046, + "tokens_seen": 1936588800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202626262626263, + "loss": 1.5527, + "theoretical_loss": 3.4396407442239036, + "tokens_seen": 1936654336 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042024242424242426, + "loss": 1.5146, + "theoretical_loss": 3.4396309780027354, + "tokens_seen": 1936719872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202222222222222, + "loss": 1.5646, + "theoretical_loss": 3.4396212122045666, + "tokens_seen": 1936785408 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.5265762209892273, + "objective/train/avg_token_score": 0.011445840820670128, + "objective/train/docs_used": 1094637, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9047353267669678, + "objective/train/original_loss": 3.971524477005005, + "objective/train/theoretical_loss": 3.4396114468293657, + "objective/train/tokens_used": 295710176, + "objective/train/unlikelihood_loss": 0.05321503430604935, + "theoretical_loss": 3.4396114468293657, + "tokens_seen": 1936850944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202020202020202, + "loss": 1.4187, + "theoretical_loss": 3.4396114468293657, + "tokens_seen": 1936850944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201818181818182, + "loss": 1.3045, + "theoretical_loss": 3.4396016818770994, + "tokens_seen": 1936916480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201616161616162, + "loss": 1.5315, + "theoretical_loss": 3.4395919173477347, + "tokens_seen": 1936982016 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042014141414141416, + "loss": 1.3532, + "theoretical_loss": 3.43958215324124, + "tokens_seen": 1937047552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201212121212121, + "loss": 1.4987, + "theoretical_loss": 3.439572389557581, + "tokens_seen": 1937113088 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042010101010101007, + "loss": 1.5557, + "theoretical_loss": 3.439562626296727, + "tokens_seen": 1937178624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200808080808081, + "loss": 1.4468, + "theoretical_loss": 3.439552863458644, + "tokens_seen": 1937244160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200606060606061, + "loss": 1.6046, + "theoretical_loss": 3.4395431010433, + "tokens_seen": 1937309696 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042004040404040405, + "loss": 1.3819, + "theoretical_loss": 3.439533339050662, + "tokens_seen": 1937375232 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420020202020202, + "loss": 1.4179, + "theoretical_loss": 3.4395235774806983, + "tokens_seen": 1937440768 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042, + "loss": 1.4089, + "theoretical_loss": 3.439513816333376, + "tokens_seen": 1937506304 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419979797979798, + "loss": 1.379, + "theoretical_loss": 3.439504055608661, + "tokens_seen": 1937571840 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419959595959596, + "loss": 1.4083, + "theoretical_loss": 3.439494295306523, + "tokens_seen": 1937637376 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041993939393939394, + "loss": 1.2468, + "theoretical_loss": 3.439484535426928, + "tokens_seen": 1937702912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199191919191919, + "loss": 1.345, + "theoretical_loss": 3.4394747759698436, + "tokens_seen": 1937768448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198989898989899, + "loss": 1.2737, + "theoretical_loss": 3.439465016935238, + "tokens_seen": 1937833984 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041987878787878787, + "loss": 1.4496, + "theoretical_loss": 3.4394552583230773, + "tokens_seen": 1937899520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198585858585859, + "loss": 1.3009, + "theoretical_loss": 3.4394455001333295, + "tokens_seen": 1937965056 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041983838383838383, + "loss": 1.6033, + "theoretical_loss": 3.439435742365963, + "tokens_seen": 1938030592 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041981818181818185, + "loss": 1.4372, + "theoretical_loss": 3.439425985020944, + "tokens_seen": 1938096128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004197979797979798, + "loss": 1.5184, + "theoretical_loss": 3.43941622809824, + "tokens_seen": 1938161664 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004197777777777778, + "loss": 1.6132, + "theoretical_loss": 3.439406471597819, + "tokens_seen": 1938227200 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041975757575757577, + "loss": 1.4743, + "theoretical_loss": 3.4393967155196483, + "tokens_seen": 1938292736 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041973737373737373, + "loss": 1.5898, + "theoretical_loss": 3.439386959863695, + "tokens_seen": 1938358272 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041971717171717174, + "loss": 1.5563, + "theoretical_loss": 3.439377204629927, + "tokens_seen": 1938423808 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.22458454966545105, + "objective/train/avg_token_score": 0.036178138107061386, + "objective/train/docs_used": 1095318, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.6959583759307861, + "objective/train/original_loss": 4.687398910522461, + "objective/train/theoretical_loss": 3.4393674498183118, + "objective/train/tokens_used": 297348576, + "objective/train/unlikelihood_loss": 0.08731012046337128, + "theoretical_loss": 3.4393674498183118, + "tokens_seen": 1938489344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196969696969697, + "loss": 1.2169, + "theoretical_loss": 3.4393674498183118, + "tokens_seen": 1938489344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196767676767677, + "loss": 1.2217, + "theoretical_loss": 3.439357695428816, + "tokens_seen": 1938554880 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041965656565656566, + "loss": 1.437, + "theoretical_loss": 3.4393479414614085, + "tokens_seen": 1938620416 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196363636363637, + "loss": 1.5371, + "theoretical_loss": 3.4393381879160554, + "tokens_seen": 1938685952 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041961616161616163, + "loss": 1.417, + "theoretical_loss": 3.439328434792725, + "tokens_seen": 1938751488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195959595959596, + "loss": 1.4628, + "theoretical_loss": 3.439318682091385, + "tokens_seen": 1938817024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195757575757576, + "loss": 1.4443, + "theoretical_loss": 3.4393089298120016, + "tokens_seen": 1938882560 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041955555555555556, + "loss": 1.3587, + "theoretical_loss": 3.4392991779545437, + "tokens_seen": 1938948096 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041953535353535357, + "loss": 1.497, + "theoretical_loss": 3.439289426518978, + "tokens_seen": 1939013632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195151515151515, + "loss": 1.6095, + "theoretical_loss": 3.4392796755052712, + "tokens_seen": 1939079168 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194949494949495, + "loss": 1.4658, + "theoretical_loss": 3.439269924913393, + "tokens_seen": 1939144704 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194747474747475, + "loss": 1.5622, + "theoretical_loss": 3.439260174743309, + "tokens_seen": 1939210240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194545454545455, + "loss": 1.5161, + "theoretical_loss": 3.439250424994988, + "tokens_seen": 1939275776 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041943434343434346, + "loss": 1.4962, + "theoretical_loss": 3.4392406756683958, + "tokens_seen": 1939341312 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194141414141414, + "loss": 1.4209, + "theoretical_loss": 3.4392309267635017, + "tokens_seen": 1939406848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193939393939394, + "loss": 1.2259, + "theoretical_loss": 3.439221178280272, + "tokens_seen": 1939472384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193737373737374, + "loss": 1.4304, + "theoretical_loss": 3.439211430218675, + "tokens_seen": 1939537920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193535353535354, + "loss": 1.4592, + "theoretical_loss": 3.439201682578678, + "tokens_seen": 1939603456 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041933333333333335, + "loss": 1.6319, + "theoretical_loss": 3.439191935360249, + "tokens_seen": 1939668992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193131313131313, + "loss": 1.351, + "theoretical_loss": 3.439182188563354, + "tokens_seen": 1939734528 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041929292929292927, + "loss": 1.6761, + "theoretical_loss": 3.4391724421879615, + "tokens_seen": 1939800064 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041927272727272733, + "loss": 1.5499, + "theoretical_loss": 3.4391626962340394, + "tokens_seen": 1939865600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004192525252525253, + "loss": 1.5205, + "theoretical_loss": 3.439152950701555, + "tokens_seen": 1939931136 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041923232323232325, + "loss": 1.4779, + "theoretical_loss": 3.439143205590475, + "tokens_seen": 1939996672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004192121212121212, + "loss": 1.4338, + "theoretical_loss": 3.439133460900768, + "tokens_seen": 1940062208 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.47330155968666077, + "objective/train/avg_token_score": 0.009856240823864937, + "objective/train/docs_used": 1096386, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5892785787582397, + "objective/train/original_loss": 4.016402721405029, + "objective/train/theoretical_loss": 3.439123716632401, + "objective/train/tokens_used": 298986976, + "objective/train/unlikelihood_loss": 0.07871534675359726, + "theoretical_loss": 3.439123716632401, + "tokens_seen": 1940127744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041919191919191916, + "loss": 1.7263, + "theoretical_loss": 3.439123716632401, + "tokens_seen": 1940127744 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191717171717172, + "loss": 1.4301, + "theoretical_loss": 3.439113972785342, + "tokens_seen": 1940193280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191515151515152, + "loss": 1.4758, + "theoretical_loss": 3.4391042293595584, + "tokens_seen": 1940258816 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041913131313131314, + "loss": 1.4596, + "theoretical_loss": 3.4390944863550175, + "tokens_seen": 1940324352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191111111111111, + "loss": 1.4683, + "theoretical_loss": 3.4390847437716863, + "tokens_seen": 1940389888 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041909090909090905, + "loss": 1.3133, + "theoretical_loss": 3.4390750016095337, + "tokens_seen": 1940455424 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190707070707071, + "loss": 1.6132, + "theoretical_loss": 3.439065259868527, + "tokens_seen": 1940520960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190505050505051, + "loss": 1.4538, + "theoretical_loss": 3.4390555185486327, + "tokens_seen": 1940586496 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041903030303030303, + "loss": 1.4229, + "theoretical_loss": 3.4390457776498193, + "tokens_seen": 1940652032 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419010101010101, + "loss": 1.363, + "theoretical_loss": 3.4390360371720545, + "tokens_seen": 1940717568 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418989898989899, + "loss": 1.4503, + "theoretical_loss": 3.439026297115305, + "tokens_seen": 1940783104 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418969696969697, + "loss": 1.3993, + "theoretical_loss": 3.4390165574795395, + "tokens_seen": 1940848640 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041894949494949497, + "loss": 1.4746, + "theoretical_loss": 3.4390068182647244, + "tokens_seen": 1940914176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189292929292929, + "loss": 1.5665, + "theoretical_loss": 3.4389970794708287, + "tokens_seen": 1940979712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189090909090909, + "loss": 1.6576, + "theoretical_loss": 3.4389873410978185, + "tokens_seen": 1941045248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188888888888889, + "loss": 1.4832, + "theoretical_loss": 3.4389776031456623, + "tokens_seen": 1941110784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188686868686869, + "loss": 1.1606, + "theoretical_loss": 3.4389678656143277, + "tokens_seen": 1941176320 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041884848484848486, + "loss": 1.5457, + "theoretical_loss": 3.4389581285037822, + "tokens_seen": 1941241856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188282828282828, + "loss": 1.2376, + "theoretical_loss": 3.4389483918139927, + "tokens_seen": 1941307392 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041880808080808083, + "loss": 1.5873, + "theoretical_loss": 3.438938655544928, + "tokens_seen": 1941372928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187878787878788, + "loss": 1.4951, + "theoretical_loss": 3.438928919696555, + "tokens_seen": 1941438464 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187676767676768, + "loss": 1.655, + "theoretical_loss": 3.4389191842688414, + "tokens_seen": 1941504000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041874747474747475, + "loss": 1.3632, + "theoretical_loss": 3.438909449261755, + "tokens_seen": 1941569536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187272727272727, + "loss": 1.4523, + "theoretical_loss": 3.4388997146752636, + "tokens_seen": 1941635072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187070707070707, + "loss": 1.3786, + "theoretical_loss": 3.4388899805093343, + "tokens_seen": 1941700608 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.43194037675857544, + "objective/train/avg_token_score": 0.020467616617679596, + "objective/train/docs_used": 1097742, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5595964193344116, + "objective/train/original_loss": 4.220158576965332, + "objective/train/theoretical_loss": 3.438880246763935, + "objective/train/tokens_used": 300625376, + "objective/train/unlikelihood_loss": 0.06359304487705231, + "theoretical_loss": 3.438880246763935, + "tokens_seen": 1941766144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186868686868687, + "loss": 1.4585, + "theoretical_loss": 3.438880246763935, + "tokens_seen": 1941766144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186666666666667, + "loss": 1.3775, + "theoretical_loss": 3.438870513439033, + "tokens_seen": 1941831680 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041864646464646464, + "loss": 1.4134, + "theoretical_loss": 3.438860780534597, + "tokens_seen": 1941897216 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041862626262626266, + "loss": 1.2068, + "theoretical_loss": 3.438851048050594, + "tokens_seen": 1941962752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186060606060606, + "loss": 1.6119, + "theoretical_loss": 3.4388413159869913, + "tokens_seen": 1942028288 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041858585858585857, + "loss": 1.4467, + "theoretical_loss": 3.438831584343757, + "tokens_seen": 1942093824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185656565656566, + "loss": 1.4658, + "theoretical_loss": 3.4388218531208583, + "tokens_seen": 1942159360 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041854545454545454, + "loss": 1.5447, + "theoretical_loss": 3.438812122318263, + "tokens_seen": 1942224896 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041852525252525255, + "loss": 1.5643, + "theoretical_loss": 3.4388023919359396, + "tokens_seen": 1942290432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185050505050505, + "loss": 1.305, + "theoretical_loss": 3.4387926619738547, + "tokens_seen": 1942355968 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041848484848484846, + "loss": 1.4774, + "theoretical_loss": 3.438782932431976, + "tokens_seen": 1942421504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184646464646465, + "loss": 1.4883, + "theoretical_loss": 3.438773203310272, + "tokens_seen": 1942487040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184444444444445, + "loss": 1.7094, + "theoretical_loss": 3.4387634746087103, + "tokens_seen": 1942552576 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041842424242424244, + "loss": 1.3692, + "theoretical_loss": 3.4387537463272575, + "tokens_seen": 1942618112 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184040404040404, + "loss": 1.5318, + "theoretical_loss": 3.438744018465882, + "tokens_seen": 1942683648 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041838383838383835, + "loss": 1.2879, + "theoretical_loss": 3.438734291024552, + "tokens_seen": 1942749184 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041836363636363637, + "loss": 1.6327, + "theoretical_loss": 3.438724564003235, + "tokens_seen": 1942814720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183434343434344, + "loss": 1.2888, + "theoretical_loss": 3.438714837401897, + "tokens_seen": 1942880256 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041832323232323233, + "loss": 1.5383, + "theoretical_loss": 3.4387051112205085, + "tokens_seen": 1942945792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183030303030303, + "loss": 1.4541, + "theoretical_loss": 3.4386953854590354, + "tokens_seen": 1943011328 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041828282828282825, + "loss": 1.4903, + "theoretical_loss": 3.4386856601174456, + "tokens_seen": 1943076864 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182626262626263, + "loss": 1.3577, + "theoretical_loss": 3.4386759351957066, + "tokens_seen": 1943142400 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041824242424242427, + "loss": 1.4549, + "theoretical_loss": 3.4386662106937873, + "tokens_seen": 1943207936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182222222222222, + "loss": 1.5026, + "theoretical_loss": 3.438656486611654, + "tokens_seen": 1943273472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182020202020202, + "loss": 1.4573, + "theoretical_loss": 3.4386467629492756, + "tokens_seen": 1943339008 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.38086509704589844, + "objective/train/avg_token_score": 0.017717378214001656, + "objective/train/docs_used": 1098461, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2602291107177734, + "objective/train/original_loss": 4.616687297821045, + "objective/train/theoretical_loss": 3.4386370397066193, + "objective/train/tokens_used": 302263776, + "objective/train/unlikelihood_loss": 0.05086660385131836, + "theoretical_loss": 3.4386370397066193, + "tokens_seen": 1943404544 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041818181818181814, + "loss": 1.4643, + "theoretical_loss": 3.4386370397066193, + "tokens_seen": 1943404544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181616161616162, + "loss": 1.4196, + "theoretical_loss": 3.4386273168836525, + "tokens_seen": 1943470080 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041814141414141416, + "loss": 1.3719, + "theoretical_loss": 3.4386175944803434, + "tokens_seen": 1943535616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181212121212121, + "loss": 1.6352, + "theoretical_loss": 3.4386078724966596, + "tokens_seen": 1943601152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181010101010101, + "loss": 1.4791, + "theoretical_loss": 3.438598150932569, + "tokens_seen": 1943666688 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041808080808080814, + "loss": 1.2838, + "theoretical_loss": 3.4385884297880387, + "tokens_seen": 1943732224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180606060606061, + "loss": 1.6374, + "theoretical_loss": 3.4385787090630373, + "tokens_seen": 1943797760 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041804040404040406, + "loss": 1.5597, + "theoretical_loss": 3.438568988757532, + "tokens_seen": 1943863296 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418020202020202, + "loss": 1.3555, + "theoretical_loss": 3.438559268871491, + "tokens_seen": 1943928832 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041799999999999997, + "loss": 1.5381, + "theoretical_loss": 3.4385495494048817, + "tokens_seen": 1943994368 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041797979797979803, + "loss": 1.6316, + "theoretical_loss": 3.4385398303576715, + "tokens_seen": 1944059904 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417959595959596, + "loss": 1.5205, + "theoretical_loss": 3.438530111729829, + "tokens_seen": 1944125440 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041793939393939395, + "loss": 1.4276, + "theoretical_loss": 3.4385203935213218, + "tokens_seen": 1944190976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179191919191919, + "loss": 1.4871, + "theoretical_loss": 3.4385106757321173, + "tokens_seen": 1944256512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178989898989899, + "loss": 1.406, + "theoretical_loss": 3.438500958362183, + "tokens_seen": 1944322048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178787878787879, + "loss": 1.6157, + "theoretical_loss": 3.438491241411487, + "tokens_seen": 1944387584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178585858585859, + "loss": 1.3643, + "theoretical_loss": 3.438481524879998, + "tokens_seen": 1944453120 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041783838383838384, + "loss": 1.5163, + "theoretical_loss": 3.438471808767683, + "tokens_seen": 1944518656 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178181818181818, + "loss": 1.5139, + "theoretical_loss": 3.4384620930745093, + "tokens_seen": 1944584192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177979797979798, + "loss": 1.6269, + "theoretical_loss": 3.438452377800445, + "tokens_seen": 1944649728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177777777777778, + "loss": 1.3211, + "theoretical_loss": 3.4384426629454587, + "tokens_seen": 1944715264 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177575757575758, + "loss": 1.4293, + "theoretical_loss": 3.438432948509517, + "tokens_seen": 1944780800 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041773737373737373, + "loss": 1.4411, + "theoretical_loss": 3.4384232344925882, + "tokens_seen": 1944846336 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041771717171717174, + "loss": 1.4302, + "theoretical_loss": 3.438413520894641, + "tokens_seen": 1944911872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176969696969697, + "loss": 1.5839, + "theoretical_loss": 3.438403807715641, + "tokens_seen": 1944977408 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.33687683939933777, + "objective/train/avg_token_score": 0.05385596305131912, + "objective/train/docs_used": 1099474, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2468277215957642, + "objective/train/original_loss": 4.72367000579834, + "objective/train/theoretical_loss": 3.4383940949555587, + "objective/train/tokens_used": 303902176, + "objective/train/unlikelihood_loss": 0.06228525936603546, + "theoretical_loss": 3.4383940949555587, + "tokens_seen": 1945042944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176767676767677, + "loss": 1.3905, + "theoretical_loss": 3.4383940949555587, + "tokens_seen": 1945042944 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041765656565656567, + "loss": 1.418, + "theoretical_loss": 3.43838438261436, + "tokens_seen": 1945108480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176363636363636, + "loss": 1.3343, + "theoretical_loss": 3.4383746706920135, + "tokens_seen": 1945174016 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041761616161616164, + "loss": 1.3567, + "theoretical_loss": 3.438364959188487, + "tokens_seen": 1945239552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175959595959596, + "loss": 1.467, + "theoretical_loss": 3.438355248103748, + "tokens_seen": 1945305088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175757575757576, + "loss": 1.5035, + "theoretical_loss": 3.4383455374377645, + "tokens_seen": 1945370624 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041755555555555556, + "loss": 1.5008, + "theoretical_loss": 3.4383358271905045, + "tokens_seen": 1945436160 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041753535353535357, + "loss": 1.2581, + "theoretical_loss": 3.4383261173619353, + "tokens_seen": 1945501696 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041751515151515153, + "loss": 1.6451, + "theoretical_loss": 3.438316407952026, + "tokens_seen": 1945567232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174949494949495, + "loss": 1.4594, + "theoretical_loss": 3.438306698960743, + "tokens_seen": 1945632768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174747474747475, + "loss": 1.5545, + "theoretical_loss": 3.4382969903880545, + "tokens_seen": 1945698304 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041745454545454545, + "loss": 1.6054, + "theoretical_loss": 3.4382872822339294, + "tokens_seen": 1945763840 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041743434343434347, + "loss": 1.6339, + "theoretical_loss": 3.4382775744983345, + "tokens_seen": 1945829376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174141414141414, + "loss": 1.3249, + "theoretical_loss": 3.4382678671812377, + "tokens_seen": 1945894912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173939393939394, + "loss": 1.4215, + "theoretical_loss": 3.438258160282607, + "tokens_seen": 1945960448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173737373737374, + "loss": 1.6104, + "theoretical_loss": 3.4382484538024105, + "tokens_seen": 1946025984 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041735353535353535, + "loss": 1.3356, + "theoretical_loss": 3.438238747740616, + "tokens_seen": 1946091520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041733333333333336, + "loss": 1.1932, + "theoretical_loss": 3.4382290420971913, + "tokens_seen": 1946157056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173131313131313, + "loss": 1.4409, + "theoretical_loss": 3.438219336872104, + "tokens_seen": 1946222592 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041729292929292927, + "loss": 1.5784, + "theoretical_loss": 3.4382096320653224, + "tokens_seen": 1946288128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172727272727273, + "loss": 1.7057, + "theoretical_loss": 3.4381999276768145, + "tokens_seen": 1946353664 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172525252525253, + "loss": 1.2891, + "theoretical_loss": 3.438190223706548, + "tokens_seen": 1946419200 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041723232323232325, + "loss": 1.4872, + "theoretical_loss": 3.43818052015449, + "tokens_seen": 1946484736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172121212121212, + "loss": 1.4779, + "theoretical_loss": 3.43817081702061, + "tokens_seen": 1946550272 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041719191919191916, + "loss": 1.3737, + "theoretical_loss": 3.4381611143048745, + "tokens_seen": 1946615808 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.4748900234699249, + "objective/train/avg_token_score": 0.007229257840663195, + "objective/train/docs_used": 1099984, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5736761093139648, + "objective/train/original_loss": 4.259389400482178, + "objective/train/theoretical_loss": 3.4381514120072523, + "objective/train/tokens_used": 305540576, + "objective/train/unlikelihood_loss": 0.05011461675167084, + "theoretical_loss": 3.4381514120072523, + "tokens_seen": 1946681344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171717171717172, + "loss": 1.6191, + "theoretical_loss": 3.4381514120072523, + "tokens_seen": 1946681344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171515151515152, + "loss": 1.5331, + "theoretical_loss": 3.4381417101277107, + "tokens_seen": 1946746880 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041713131313131314, + "loss": 1.5493, + "theoretical_loss": 3.438132008666218, + "tokens_seen": 1946812416 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171111111111111, + "loss": 1.3735, + "theoretical_loss": 3.438122307622742, + "tokens_seen": 1946877952 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041709090909090906, + "loss": 1.5348, + "theoretical_loss": 3.43811260699725, + "tokens_seen": 1946943488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170707070707071, + "loss": 1.5874, + "theoretical_loss": 3.4381029067897115, + "tokens_seen": 1947009024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170505050505051, + "loss": 1.4854, + "theoretical_loss": 3.438093207000093, + "tokens_seen": 1947074560 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041703030303030304, + "loss": 1.6018, + "theoretical_loss": 3.4380835076283622, + "tokens_seen": 1947140096 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417010101010101, + "loss": 1.3937, + "theoretical_loss": 3.4380738086744884, + "tokens_seen": 1947205632 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041698989898989895, + "loss": 1.3372, + "theoretical_loss": 3.4380641101384386, + "tokens_seen": 1947271168 + }, + { + "epoch": 0.17, + "learning_rate": 0.000416969696969697, + "loss": 1.4951, + "theoretical_loss": 3.4380544120201812, + "tokens_seen": 1947336704 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041694949494949497, + "loss": 1.2962, + "theoretical_loss": 3.438044714319684, + "tokens_seen": 1947402240 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041692929292929293, + "loss": 1.4141, + "theoretical_loss": 3.4380350170369143, + "tokens_seen": 1947467776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169090909090909, + "loss": 1.3446, + "theoretical_loss": 3.438025320171841, + "tokens_seen": 1947533312 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168888888888889, + "loss": 1.469, + "theoretical_loss": 3.438015623724432, + "tokens_seen": 1947598848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168686868686869, + "loss": 1.516, + "theoretical_loss": 3.4380059276946544, + "tokens_seen": 1947664384 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041684848484848486, + "loss": 1.4418, + "theoretical_loss": 3.437996232082477, + "tokens_seen": 1947729920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168282828282828, + "loss": 1.5376, + "theoretical_loss": 3.437986536887867, + "tokens_seen": 1947795456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168080808080808, + "loss": 1.396, + "theoretical_loss": 3.437976842110793, + "tokens_seen": 1947860992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167878787878788, + "loss": 1.2724, + "theoretical_loss": 3.4379671477512233, + "tokens_seen": 1947926528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167676767676768, + "loss": 1.306, + "theoretical_loss": 3.4379574538091244, + "tokens_seen": 1947992064 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041674747474747476, + "loss": 1.656, + "theoretical_loss": 3.437947760284466, + "tokens_seen": 1948057600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167272727272727, + "loss": 1.4577, + "theoretical_loss": 3.437938067177215, + "tokens_seen": 1948123136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167070707070707, + "loss": 1.3489, + "theoretical_loss": 3.43792837448734, + "tokens_seen": 1948188672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004166868686868687, + "loss": 1.574, + "theoretical_loss": 3.437918682214808, + "tokens_seen": 1948254208 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.3907013535499573, + "objective/train/avg_token_score": 0.006235840264707804, + "objective/train/docs_used": 1101410, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.310508370399475, + "objective/train/original_loss": 4.164927005767822, + "objective/train/theoretical_loss": 3.4379089903595883, + "objective/train/tokens_used": 307178976, + "objective/train/unlikelihood_loss": 0.08828544616699219, + "theoretical_loss": 3.4379089903595883, + "tokens_seen": 1948319744 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004166666666666667, + "loss": 1.4446, + "theoretical_loss": 3.4379089903595883, + "tokens_seen": 1948319744 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041664646464646465, + "loss": 1.258, + "theoretical_loss": 3.437899298921648, + "tokens_seen": 1948385280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166262626262626, + "loss": 1.2467, + "theoretical_loss": 3.4378896079009555, + "tokens_seen": 1948450816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166060606060606, + "loss": 1.2485, + "theoretical_loss": 3.437879917297479, + "tokens_seen": 1948516352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165858585858586, + "loss": 1.4547, + "theoretical_loss": 3.437870227111186, + "tokens_seen": 1948581888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165656565656566, + "loss": 1.5381, + "theoretical_loss": 3.4378605373420443, + "tokens_seen": 1948647424 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041654545454545454, + "loss": 1.4898, + "theoretical_loss": 3.4378508479900227, + "tokens_seen": 1948712960 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041652525252525255, + "loss": 1.6897, + "theoretical_loss": 3.437841159055089, + "tokens_seen": 1948778496 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165050505050505, + "loss": 1.5336, + "theoretical_loss": 3.4378314705372106, + "tokens_seen": 1948844032 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041648484848484847, + "loss": 1.4052, + "theoretical_loss": 3.437821782436356, + "tokens_seen": 1948909568 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164646464646465, + "loss": 1.5426, + "theoretical_loss": 3.4378120947524935, + "tokens_seen": 1948975104 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041644444444444444, + "loss": 1.5688, + "theoretical_loss": 3.4378024074855906, + "tokens_seen": 1949040640 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041642424242424245, + "loss": 1.5614, + "theoretical_loss": 3.437792720635616, + "tokens_seen": 1949106176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164040404040404, + "loss": 1.4658, + "theoretical_loss": 3.437783034202537, + "tokens_seen": 1949171712 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041638383838383836, + "loss": 1.407, + "theoretical_loss": 3.4377733481863215, + "tokens_seen": 1949237248 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041636363636363637, + "loss": 1.343, + "theoretical_loss": 3.4377636625869386, + "tokens_seen": 1949302784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163434343434344, + "loss": 1.5019, + "theoretical_loss": 3.4377539774043555, + "tokens_seen": 1949368320 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041632323232323234, + "loss": 1.4887, + "theoretical_loss": 3.4377442926385404, + "tokens_seen": 1949433856 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163030303030303, + "loss": 1.5129, + "theoretical_loss": 3.4377346082894618, + "tokens_seen": 1949499392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162828282828283, + "loss": 1.3991, + "theoretical_loss": 3.437724924357087, + "tokens_seen": 1949564928 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041626262626262626, + "loss": 1.6064, + "theoretical_loss": 3.437715240841385, + "tokens_seen": 1949630464 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162424242424243, + "loss": 1.5792, + "theoretical_loss": 3.437705557742323, + "tokens_seen": 1949696000 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041622222222222223, + "loss": 1.6014, + "theoretical_loss": 3.4376958750598696, + "tokens_seen": 1949761536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162020202020202, + "loss": 1.515, + "theoretical_loss": 3.437686192793992, + "tokens_seen": 1949827072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161818181818182, + "loss": 1.3131, + "theoretical_loss": 3.4376765109446596, + "tokens_seen": 1949892608 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.24596774578094482, + "objective/train/avg_token_score": 0.017320265993475914, + "objective/train/docs_used": 1102181, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8094052076339722, + "objective/train/original_loss": 4.91034460067749, + "objective/train/theoretical_loss": 3.4376668295118398, + "objective/train/tokens_used": 308817376, + "objective/train/unlikelihood_loss": 0.06164025887846947, + "theoretical_loss": 3.4376668295118398, + "tokens_seen": 1949958144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161616161616162, + "loss": 1.3284, + "theoretical_loss": 3.4376668295118398, + "tokens_seen": 1949958144 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041614141414141417, + "loss": 1.476, + "theoretical_loss": 3.4376571484955005, + "tokens_seen": 1950023680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161212121212121, + "loss": 1.4625, + "theoretical_loss": 3.4376474678956104, + "tokens_seen": 1950089216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161010101010101, + "loss": 1.2883, + "theoretical_loss": 3.437637787712137, + "tokens_seen": 1950154752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160808080808081, + "loss": 1.4984, + "theoretical_loss": 3.4376281079450486, + "tokens_seen": 1950220288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160606060606061, + "loss": 1.6164, + "theoretical_loss": 3.437618428594313, + "tokens_seen": 1950285824 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041604040404040406, + "loss": 1.4649, + "theoretical_loss": 3.4376087496598986, + "tokens_seen": 1950351360 + }, + { + "epoch": 0.18, + "learning_rate": 0.000416020202020202, + "loss": 1.5006, + "theoretical_loss": 3.437599071141774, + "tokens_seen": 1950416896 + }, + { + "epoch": 0.18, + "learning_rate": 0.000416, + "loss": 1.4269, + "theoretical_loss": 3.4375893930399064, + "tokens_seen": 1950482432 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041597979797979804, + "loss": 1.3195, + "theoretical_loss": 3.4375797153542647, + "tokens_seen": 1950547968 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415959595959596, + "loss": 1.5313, + "theoretical_loss": 3.437570038084816, + "tokens_seen": 1950613504 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041593939393939395, + "loss": 1.3833, + "theoretical_loss": 3.4375603612315295, + "tokens_seen": 1950679040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159191919191919, + "loss": 1.5523, + "theoretical_loss": 3.437550684794373, + "tokens_seen": 1950744576 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041589898989898987, + "loss": 1.3026, + "theoretical_loss": 3.4375410087733145, + "tokens_seen": 1950810112 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041587878787878793, + "loss": 1.1543, + "theoretical_loss": 3.437531333168322, + "tokens_seen": 1950875648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158585858585859, + "loss": 1.4297, + "theoretical_loss": 3.4375216579793637, + "tokens_seen": 1950941184 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041583838383838385, + "loss": 1.3946, + "theoretical_loss": 3.4375119832064076, + "tokens_seen": 1951006720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158181818181818, + "loss": 1.637, + "theoretical_loss": 3.4375023088494228, + "tokens_seen": 1951072256 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041579797979797976, + "loss": 1.4739, + "theoretical_loss": 3.4374926349083763, + "tokens_seen": 1951137792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157777777777778, + "loss": 1.1972, + "theoretical_loss": 3.4374829613832363, + "tokens_seen": 1951203328 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157575757575758, + "loss": 1.2674, + "theoretical_loss": 3.4374732882739716, + "tokens_seen": 1951268864 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041573737373737374, + "loss": 1.2435, + "theoretical_loss": 3.43746361558055, + "tokens_seen": 1951334400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157171717171717, + "loss": 1.283, + "theoretical_loss": 3.4374539433029394, + "tokens_seen": 1951399936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156969696969697, + "loss": 1.5095, + "theoretical_loss": 3.437444271441109, + "tokens_seen": 1951465472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156767676767677, + "loss": 1.5597, + "theoretical_loss": 3.4374345999950253, + "tokens_seen": 1951531008 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.5144794583320618, + "objective/train/avg_token_score": 0.006237310823053122, + "objective/train/docs_used": 1103575, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6771392822265625, + "objective/train/original_loss": 3.8277225494384766, + "objective/train/theoretical_loss": 3.437424928964658, + "objective/train/tokens_used": 310455776, + "objective/train/unlikelihood_loss": 0.050984472036361694, + "theoretical_loss": 3.437424928964658, + "tokens_seen": 1951596544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156565656565657, + "loss": 1.5589, + "theoretical_loss": 3.437424928964658, + "tokens_seen": 1951596544 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041563636363636363, + "loss": 1.3017, + "theoretical_loss": 3.437415258349975, + "tokens_seen": 1951662080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156161616161616, + "loss": 1.5017, + "theoretical_loss": 3.437405588150943, + "tokens_seen": 1951727616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155959595959596, + "loss": 1.3667, + "theoretical_loss": 3.437395918367532, + "tokens_seen": 1951793152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155757575757576, + "loss": 1.6398, + "theoretical_loss": 3.43738624899971, + "tokens_seen": 1951858688 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041555555555555557, + "loss": 1.5077, + "theoretical_loss": 3.437376580047444, + "tokens_seen": 1951924224 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155353535353535, + "loss": 1.6662, + "theoretical_loss": 3.437366911510703, + "tokens_seen": 1951989760 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041551515151515154, + "loss": 1.524, + "theoretical_loss": 3.437357243389455, + "tokens_seen": 1952055296 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154949494949495, + "loss": 1.2516, + "theoretical_loss": 3.437347575683668, + "tokens_seen": 1952120832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154747474747475, + "loss": 1.4622, + "theoretical_loss": 3.437337908393311, + "tokens_seen": 1952186368 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041545454545454546, + "loss": 1.5639, + "theoretical_loss": 3.437328241518352, + "tokens_seen": 1952251904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154343434343434, + "loss": 1.4582, + "theoretical_loss": 3.4373185750587583, + "tokens_seen": 1952317440 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041541414141414143, + "loss": 1.2993, + "theoretical_loss": 3.4373089090144986, + "tokens_seen": 1952382976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153939393939394, + "loss": 1.689, + "theoretical_loss": 3.4372992433855414, + "tokens_seen": 1952448512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153737373737374, + "loss": 1.5325, + "theoretical_loss": 3.4372895781718547, + "tokens_seen": 1952514048 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041535353535353535, + "loss": 1.5222, + "theoretical_loss": 3.437279913373407, + "tokens_seen": 1952579584 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041533333333333336, + "loss": 1.2857, + "theoretical_loss": 3.437270248990166, + "tokens_seen": 1952645120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153131313131313, + "loss": 1.4812, + "theoretical_loss": 3.4372605850221003, + "tokens_seen": 1952710656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152929292929293, + "loss": 1.4163, + "theoretical_loss": 3.437250921469178, + "tokens_seen": 1952776192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152727272727273, + "loss": 1.58, + "theoretical_loss": 3.437241258331367, + "tokens_seen": 1952841728 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041525252525252525, + "loss": 1.5932, + "theoretical_loss": 3.437231595608636, + "tokens_seen": 1952907264 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041523232323232326, + "loss": 1.3163, + "theoretical_loss": 3.4372219333009535, + "tokens_seen": 1952972800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152121212121212, + "loss": 1.4865, + "theoretical_loss": 3.437212271408287, + "tokens_seen": 1953038336 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041519191919191917, + "loss": 1.3725, + "theoretical_loss": 3.4372026099306057, + "tokens_seen": 1953103872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151717171717172, + "loss": 1.4658, + "theoretical_loss": 3.4371929488678767, + "tokens_seen": 1953169408 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.3589320480823517, + "objective/train/avg_token_score": 0.013197287917137146, + "objective/train/docs_used": 1104173, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2983180284500122, + "objective/train/original_loss": 4.760884761810303, + "objective/train/theoretical_loss": 3.4371832882200692, + "objective/train/tokens_used": 312094176, + "objective/train/unlikelihood_loss": 0.052073884755373, + "theoretical_loss": 3.4371832882200692, + "tokens_seen": 1953234944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151515151515152, + "loss": 1.3511, + "theoretical_loss": 3.4371832882200692, + "tokens_seen": 1953234944 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041513131313131315, + "loss": 1.3083, + "theoretical_loss": 3.4371736279871508, + "tokens_seen": 1953300480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151111111111111, + "loss": 1.6078, + "theoretical_loss": 3.43716396816909, + "tokens_seen": 1953366016 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041509090909090906, + "loss": 1.5163, + "theoretical_loss": 3.4371543087658556, + "tokens_seen": 1953431552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150707070707071, + "loss": 1.3672, + "theoretical_loss": 3.437144649777415, + "tokens_seen": 1953497088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150505050505051, + "loss": 1.3603, + "theoretical_loss": 3.4371349912037372, + "tokens_seen": 1953562624 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041503030303030304, + "loss": 1.472, + "theoretical_loss": 3.43712533304479, + "tokens_seen": 1953628160 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415010101010101, + "loss": 1.4868, + "theoretical_loss": 3.437115675300542, + "tokens_seen": 1953693696 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041498989898989896, + "loss": 1.3721, + "theoretical_loss": 3.4371060179709607, + "tokens_seen": 1953759232 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414969696969697, + "loss": 1.5007, + "theoretical_loss": 3.4370963610560157, + "tokens_seen": 1953824768 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414949494949495, + "loss": 1.3869, + "theoretical_loss": 3.437086704555674, + "tokens_seen": 1953890304 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041492929292929293, + "loss": 1.5926, + "theoretical_loss": 3.437077048469905, + "tokens_seen": 1953955840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149090909090909, + "loss": 1.5715, + "theoretical_loss": 3.437067392798676, + "tokens_seen": 1954021376 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041488888888888885, + "loss": 1.688, + "theoretical_loss": 3.4370577375419566, + "tokens_seen": 1954086912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148686868686869, + "loss": 1.3722, + "theoretical_loss": 3.4370480826997136, + "tokens_seen": 1954152448 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041484848484848487, + "loss": 1.4423, + "theoretical_loss": 3.437038428271916, + "tokens_seen": 1954217984 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041482828282828283, + "loss": 1.4912, + "theoretical_loss": 3.4370287742585326, + "tokens_seen": 1954283520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148080808080808, + "loss": 1.453, + "theoretical_loss": 3.4370191206595306, + "tokens_seen": 1954349056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147878787878788, + "loss": 1.473, + "theoretical_loss": 3.437009467474879, + "tokens_seen": 1954414592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147676767676768, + "loss": 1.533, + "theoretical_loss": 3.4369998147045466, + "tokens_seen": 1954480128 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041474747474747476, + "loss": 1.1881, + "theoretical_loss": 3.4369901623485006, + "tokens_seen": 1954545664 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147272727272727, + "loss": 1.4963, + "theoretical_loss": 3.43698051040671, + "tokens_seen": 1954611200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147070707070707, + "loss": 1.501, + "theoretical_loss": 3.4369708588791434, + "tokens_seen": 1954676736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146868686868687, + "loss": 1.4789, + "theoretical_loss": 3.4369612077657687, + "tokens_seen": 1954742272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146666666666667, + "loss": 1.4487, + "theoretical_loss": 3.4369515570665543, + "tokens_seen": 1954807808 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.42155423760414124, + "objective/train/avg_token_score": 0.007324489299207926, + "objective/train/docs_used": 1105519, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2957619428634644, + "objective/train/original_loss": 4.201072692871094, + "objective/train/theoretical_loss": 3.4369419067814686, + "objective/train/tokens_used": 313732576, + "objective/train/unlikelihood_loss": 0.06063661724328995, + "theoretical_loss": 3.4369419067814686, + "tokens_seen": 1954873344 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041464646464646466, + "loss": 1.3553, + "theoretical_loss": 3.4369419067814686, + "tokens_seen": 1954873344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146262626262626, + "loss": 1.3954, + "theoretical_loss": 3.4369322569104797, + "tokens_seen": 1954938880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146060606060606, + "loss": 1.4729, + "theoretical_loss": 3.4369226074535564, + "tokens_seen": 1955004416 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041458585858585863, + "loss": 1.5528, + "theoretical_loss": 3.436912958410667, + "tokens_seen": 1955069952 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145656565656566, + "loss": 1.2495, + "theoretical_loss": 3.436903309781779, + "tokens_seen": 1955135488 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041454545454545455, + "loss": 1.6581, + "theoretical_loss": 3.436893661566862, + "tokens_seen": 1955201024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145252525252525, + "loss": 1.3339, + "theoretical_loss": 3.4368840137658836, + "tokens_seen": 1955266560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145050505050505, + "loss": 1.4273, + "theoretical_loss": 3.436874366378812, + "tokens_seen": 1955332096 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041448484848484853, + "loss": 1.1655, + "theoretical_loss": 3.4368647194056168, + "tokens_seen": 1955397632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144646464646465, + "loss": 1.4596, + "theoretical_loss": 3.436855072846265, + "tokens_seen": 1955463168 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041444444444444444, + "loss": 1.295, + "theoretical_loss": 3.4368454267007253, + "tokens_seen": 1955528704 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041442424242424245, + "loss": 1.5501, + "theoretical_loss": 3.4368357809689667, + "tokens_seen": 1955594240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144040404040404, + "loss": 1.4875, + "theoretical_loss": 3.436826135650957, + "tokens_seen": 1955659776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143838383838384, + "loss": 1.4391, + "theoretical_loss": 3.436816490746665, + "tokens_seen": 1955725312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143636363636364, + "loss": 1.436, + "theoretical_loss": 3.436806846256058, + "tokens_seen": 1955790848 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041434343434343433, + "loss": 1.6272, + "theoretical_loss": 3.436797202179106, + "tokens_seen": 1955856384 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041432323232323235, + "loss": 1.5892, + "theoretical_loss": 3.4367875585157766, + "tokens_seen": 1955921920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143030303030303, + "loss": 1.5377, + "theoretical_loss": 3.4367779152660383, + "tokens_seen": 1955987456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142828282828283, + "loss": 1.1497, + "theoretical_loss": 3.436768272429859, + "tokens_seen": 1956052992 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041426262626262627, + "loss": 1.3424, + "theoretical_loss": 3.4367586300072075, + "tokens_seen": 1956118528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142424242424243, + "loss": 1.6651, + "theoretical_loss": 3.4367489879980524, + "tokens_seen": 1956184064 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041422222222222224, + "loss": 1.3298, + "theoretical_loss": 3.436739346402362, + "tokens_seen": 1956249600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142020202020202, + "loss": 1.48, + "theoretical_loss": 3.436729705220105, + "tokens_seen": 1956315136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141818181818182, + "loss": 1.5618, + "theoretical_loss": 3.436720064451249, + "tokens_seen": 1956380672 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041416161616161616, + "loss": 1.265, + "theoretical_loss": 3.4367104240957635, + "tokens_seen": 1956446208 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.32251954078674316, + "objective/train/avg_token_score": 0.009885277599096298, + "objective/train/docs_used": 1106102, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.99889075756073, + "objective/train/original_loss": 4.139228820800781, + "objective/train/theoretical_loss": 3.436700784153616, + "objective/train/tokens_used": 315370976, + "objective/train/unlikelihood_loss": 0.07105885446071625, + "theoretical_loss": 3.436700784153616, + "tokens_seen": 1956511744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141414141414142, + "loss": 1.3508, + "theoretical_loss": 3.436700784153616, + "tokens_seen": 1956511744 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041412121212121213, + "loss": 1.4683, + "theoretical_loss": 3.4366911446247754, + "tokens_seen": 1956577280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141010101010101, + "loss": 1.5079, + "theoretical_loss": 3.43668150550921, + "tokens_seen": 1956642816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140808080808081, + "loss": 1.5571, + "theoretical_loss": 3.4366718668068885, + "tokens_seen": 1956708352 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041406060606060606, + "loss": 1.4941, + "theoretical_loss": 3.436662228517779, + "tokens_seen": 1956773888 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041404040404040407, + "loss": 1.5562, + "theoretical_loss": 3.4366525906418497, + "tokens_seen": 1956839424 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414020202020202, + "loss": 1.7608, + "theoretical_loss": 3.43664295317907, + "tokens_seen": 1956904960 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414, + "loss": 1.1626, + "theoretical_loss": 3.4366333161294076, + "tokens_seen": 1956970496 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413979797979798, + "loss": 1.2521, + "theoretical_loss": 3.436623679492831, + "tokens_seen": 1957036032 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413959595959596, + "loss": 1.5362, + "theoretical_loss": 3.436614043269309, + "tokens_seen": 1957101568 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041393939393939396, + "loss": 1.3286, + "theoretical_loss": 3.43660440745881, + "tokens_seen": 1957167104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139191919191919, + "loss": 1.4399, + "theoretical_loss": 3.4365947720613015, + "tokens_seen": 1957232640 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041389898989898987, + "loss": 1.478, + "theoretical_loss": 3.4365851370767535, + "tokens_seen": 1957298176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138787878787879, + "loss": 1.4755, + "theoretical_loss": 3.436575502505134, + "tokens_seen": 1957363712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138585858585859, + "loss": 1.505, + "theoretical_loss": 3.4365658683464106, + "tokens_seen": 1957429248 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041383838383838385, + "loss": 1.4277, + "theoretical_loss": 3.4365562346005527, + "tokens_seen": 1957494784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138181818181818, + "loss": 1.2607, + "theoretical_loss": 3.436546601267529, + "tokens_seen": 1957560320 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041379797979797977, + "loss": 1.4686, + "theoretical_loss": 3.4365369683473066, + "tokens_seen": 1957625856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041377777777777783, + "loss": 1.1683, + "theoretical_loss": 3.4365273358398554, + "tokens_seen": 1957691392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137575757575758, + "loss": 1.3763, + "theoretical_loss": 3.4365177037451433, + "tokens_seen": 1957756928 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041373737373737374, + "loss": 1.1928, + "theoretical_loss": 3.4365080720631394, + "tokens_seen": 1957822464 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137171717171717, + "loss": 1.5801, + "theoretical_loss": 3.4364984407938115, + "tokens_seen": 1957888000 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041369696969696966, + "loss": 1.2228, + "theoretical_loss": 3.436488809937128, + "tokens_seen": 1957953536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136767676767677, + "loss": 1.3379, + "theoretical_loss": 3.4364791794930576, + "tokens_seen": 1958019072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136565656565657, + "loss": 1.3664, + "theoretical_loss": 3.4364695494615694, + "tokens_seen": 1958084608 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.4745234549045563, + "objective/train/avg_token_score": 0.011926386505365372, + "objective/train/docs_used": 1106840, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.578378438949585, + "objective/train/original_loss": 4.3692216873168945, + "objective/train/theoretical_loss": 3.4364599198426307, + "objective/train/tokens_used": 317009376, + "objective/train/unlikelihood_loss": 0.04125186428427696, + "theoretical_loss": 3.4364599198426307, + "tokens_seen": 1958150144 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041363636363636364, + "loss": 1.6483, + "theoretical_loss": 3.4364599198426307, + "tokens_seen": 1958150144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136161616161616, + "loss": 1.5616, + "theoretical_loss": 3.4364502906362118, + "tokens_seen": 1958215680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135959595959596, + "loss": 1.3994, + "theoretical_loss": 3.4364406618422794, + "tokens_seen": 1958281216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135757575757576, + "loss": 1.3587, + "theoretical_loss": 3.436431033460803, + "tokens_seen": 1958346752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135555555555556, + "loss": 1.526, + "theoretical_loss": 3.4364214054917515, + "tokens_seen": 1958412288 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041353535353535353, + "loss": 1.2816, + "theoretical_loss": 3.436411777935092, + "tokens_seen": 1958477824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135151515151515, + "loss": 1.3146, + "theoretical_loss": 3.4364021507907943, + "tokens_seen": 1958543360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134949494949495, + "loss": 1.3607, + "theoretical_loss": 3.4363925240588267, + "tokens_seen": 1958608896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134747474747475, + "loss": 1.4933, + "theoretical_loss": 3.4363828977391577, + "tokens_seen": 1958674432 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041345454545454547, + "loss": 1.4624, + "theoretical_loss": 3.436373271831755, + "tokens_seen": 1958739968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134343434343434, + "loss": 1.4388, + "theoretical_loss": 3.4363636463365888, + "tokens_seen": 1958805504 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041341414141414143, + "loss": 1.3727, + "theoretical_loss": 3.436354021253626, + "tokens_seen": 1958871040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133939393939394, + "loss": 1.3955, + "theoretical_loss": 3.436344396582837, + "tokens_seen": 1958936576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133737373737374, + "loss": 1.3918, + "theoretical_loss": 3.436334772324188, + "tokens_seen": 1959002112 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041335353535353536, + "loss": 1.4894, + "theoretical_loss": 3.43632514847765, + "tokens_seen": 1959067648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133333333333333, + "loss": 1.4914, + "theoretical_loss": 3.4363155250431894, + "tokens_seen": 1959133184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133131313131313, + "loss": 1.487, + "theoretical_loss": 3.4363059020207762, + "tokens_seen": 1959198720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132929292929293, + "loss": 1.4438, + "theoretical_loss": 3.436296279410379, + "tokens_seen": 1959264256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132727272727273, + "loss": 1.4592, + "theoretical_loss": 3.436286657211965, + "tokens_seen": 1959329792 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041325252525252525, + "loss": 1.5674, + "theoretical_loss": 3.4362770354255043, + "tokens_seen": 1959395328 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041323232323232326, + "loss": 1.5201, + "theoretical_loss": 3.436267414050965, + "tokens_seen": 1959460864 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132121212121212, + "loss": 1.3334, + "theoretical_loss": 3.4362577930883154, + "tokens_seen": 1959526400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131919191919192, + "loss": 1.6545, + "theoretical_loss": 3.436248172537524, + "tokens_seen": 1959591936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131717171717172, + "loss": 1.5059, + "theoretical_loss": 3.43623855239856, + "tokens_seen": 1959657472 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041315151515151514, + "loss": 1.4031, + "theoretical_loss": 3.4362289326713915, + "tokens_seen": 1959723008 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.34634652733802795, + "objective/train/avg_token_score": 0.02731332927942276, + "objective/train/docs_used": 1108280, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.078272819519043, + "objective/train/original_loss": 4.436617851257324, + "objective/train/theoretical_loss": 3.4362193133559877, + "objective/train/tokens_used": 318647776, + "objective/train/unlikelihood_loss": 0.09181275963783264, + "theoretical_loss": 3.4362193133559877, + "tokens_seen": 1959788544 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041313131313131315, + "loss": 1.3301, + "theoretical_loss": 3.4362193133559877, + "tokens_seen": 1959788544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131111111111111, + "loss": 1.2653, + "theoretical_loss": 3.4362096944523164, + "tokens_seen": 1959854080 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041309090909090907, + "loss": 1.5523, + "theoretical_loss": 3.436200075960347, + "tokens_seen": 1959919616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130707070707071, + "loss": 1.5079, + "theoretical_loss": 3.4361904578800466, + "tokens_seen": 1959985152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130505050505051, + "loss": 1.3166, + "theoretical_loss": 3.436180840211386, + "tokens_seen": 1960050688 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041303030303030305, + "loss": 1.5395, + "theoretical_loss": 3.436171222954332, + "tokens_seen": 1960116224 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413010101010101, + "loss": 1.2939, + "theoretical_loss": 3.436161606108855, + "tokens_seen": 1960181760 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041298989898989896, + "loss": 1.4107, + "theoretical_loss": 3.436151989674922, + "tokens_seen": 1960247296 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041296969696969697, + "loss": 1.3592, + "theoretical_loss": 3.436142373652502, + "tokens_seen": 1960312832 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412949494949495, + "loss": 1.5549, + "theoretical_loss": 3.4361327580415644, + "tokens_seen": 1960378368 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041292929292929294, + "loss": 1.5716, + "theoretical_loss": 3.436123142842077, + "tokens_seen": 1960443904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129090909090909, + "loss": 1.3391, + "theoretical_loss": 3.4361135280540087, + "tokens_seen": 1960509440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128888888888889, + "loss": 1.4607, + "theoretical_loss": 3.436103913677328, + "tokens_seen": 1960574976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128686868686869, + "loss": 1.35, + "theoretical_loss": 3.4360942997120043, + "tokens_seen": 1960640512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128484848484849, + "loss": 1.1904, + "theoretical_loss": 3.436084686158005, + "tokens_seen": 1960706048 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041282828282828283, + "loss": 1.216, + "theoretical_loss": 3.4360750730153002, + "tokens_seen": 1960771584 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128080808080808, + "loss": 1.3311, + "theoretical_loss": 3.4360654602838574, + "tokens_seen": 1960837120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127878787878788, + "loss": 1.4853, + "theoretical_loss": 3.4360558479636456, + "tokens_seen": 1960902656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127676767676768, + "loss": 1.6454, + "theoretical_loss": 3.4360462360546333, + "tokens_seen": 1960968192 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041274747474747477, + "loss": 1.2993, + "theoretical_loss": 3.43603662455679, + "tokens_seen": 1961033728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127272727272727, + "loss": 1.2265, + "theoretical_loss": 3.436027013470083, + "tokens_seen": 1961099264 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127070707070707, + "loss": 1.3192, + "theoretical_loss": 3.436017402794482, + "tokens_seen": 1961164800 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041268686868686875, + "loss": 1.6334, + "theoretical_loss": 3.4360077925299555, + "tokens_seen": 1961230336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126666666666667, + "loss": 1.3333, + "theoretical_loss": 3.435998182676472, + "tokens_seen": 1961295872 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041264646464646466, + "loss": 1.3261, + "theoretical_loss": 3.4359885732340008, + "tokens_seen": 1961361408 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.43071848154067993, + "objective/train/avg_token_score": 0.014302323572337627, + "objective/train/docs_used": 1108943, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3354324102401733, + "objective/train/original_loss": 4.234708309173584, + "objective/train/theoretical_loss": 3.4359789642025094, + "objective/train/tokens_used": 320286176, + "objective/train/unlikelihood_loss": 0.055439990013837814, + "theoretical_loss": 3.4359789642025094, + "tokens_seen": 1961426944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126262626262626, + "loss": 1.5046, + "theoretical_loss": 3.4359789642025094, + "tokens_seen": 1961426944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126060606060606, + "loss": 1.2966, + "theoretical_loss": 3.4359693555819675, + "tokens_seen": 1961492480 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041258585858585864, + "loss": 1.4952, + "theoretical_loss": 3.435959747372343, + "tokens_seen": 1961558016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125656565656566, + "loss": 1.3245, + "theoretical_loss": 3.4359501395736056, + "tokens_seen": 1961623552 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041254545454545455, + "loss": 1.3205, + "theoretical_loss": 3.4359405321857235, + "tokens_seen": 1961689088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125252525252525, + "loss": 1.2586, + "theoretical_loss": 3.4359309252086647, + "tokens_seen": 1961754624 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041250505050505047, + "loss": 1.124, + "theoretical_loss": 3.435921318642399, + "tokens_seen": 1961820160 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041248484848484853, + "loss": 1.2376, + "theoretical_loss": 3.4359117124868948, + "tokens_seen": 1961885696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124646464646465, + "loss": 1.5335, + "theoretical_loss": 3.4359021067421205, + "tokens_seen": 1961951232 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041244444444444445, + "loss": 1.5446, + "theoretical_loss": 3.435892501408045, + "tokens_seen": 1962016768 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124242424242424, + "loss": 1.2378, + "theoretical_loss": 3.435882896484637, + "tokens_seen": 1962082304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124040404040404, + "loss": 1.3928, + "theoretical_loss": 3.4358732919718653, + "tokens_seen": 1962147840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123838383838384, + "loss": 1.5574, + "theoretical_loss": 3.435863687869699, + "tokens_seen": 1962213376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123636363636364, + "loss": 1.2501, + "theoretical_loss": 3.4358540841781062, + "tokens_seen": 1962278912 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041234343434343434, + "loss": 1.3118, + "theoretical_loss": 3.435844480897056, + "tokens_seen": 1962344448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123232323232323, + "loss": 1.3476, + "theoretical_loss": 3.435834878026516, + "tokens_seen": 1962409984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123030303030303, + "loss": 1.2193, + "theoretical_loss": 3.435825275566457, + "tokens_seen": 1962475520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122828282828283, + "loss": 1.7092, + "theoretical_loss": 3.4358156735168466, + "tokens_seen": 1962541056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122626262626263, + "loss": 1.3348, + "theoretical_loss": 3.4358060718776535, + "tokens_seen": 1962606592 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041224242424242423, + "loss": 1.4579, + "theoretical_loss": 3.435796470648846, + "tokens_seen": 1962672128 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041222222222222224, + "loss": 1.4916, + "theoretical_loss": 3.4357868698303946, + "tokens_seen": 1962737664 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122020202020202, + "loss": 1.5911, + "theoretical_loss": 3.435777269422266, + "tokens_seen": 1962803200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121818181818182, + "loss": 1.3839, + "theoretical_loss": 3.4357676694244303, + "tokens_seen": 1962868736 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041216161616161617, + "loss": 1.4081, + "theoretical_loss": 3.4357580698368557, + "tokens_seen": 1962934272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121414141414141, + "loss": 1.2923, + "theoretical_loss": 3.435748470659511, + "tokens_seen": 1962999808 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.3285679519176483, + "objective/train/avg_token_score": 0.03589082136750221, + "objective/train/docs_used": 1110285, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1867942810058594, + "objective/train/original_loss": 5.2249436378479, + "objective/train/theoretical_loss": 3.4357388718923656, + "objective/train/tokens_used": 321924576, + "objective/train/unlikelihood_loss": 0.03941744193434715, + "theoretical_loss": 3.4357388718923656, + "tokens_seen": 1963065344 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041212121212121214, + "loss": 1.4527, + "theoretical_loss": 3.4357388718923656, + "tokens_seen": 1963065344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121010101010101, + "loss": 1.3501, + "theoretical_loss": 3.4357292735353875, + "tokens_seen": 1963130880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004120808080808081, + "loss": 1.4519, + "theoretical_loss": 3.4357196755885457, + "tokens_seen": 1963196416 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041206060606060606, + "loss": 1.4366, + "theoretical_loss": 3.435710078051809, + "tokens_seen": 1963261952 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041204040404040407, + "loss": 1.3558, + "theoretical_loss": 3.435700480925146, + "tokens_seen": 1963327488 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041202020202020203, + "loss": 1.6353, + "theoretical_loss": 3.435690884208526, + "tokens_seen": 1963393024 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412, + "loss": 1.477, + "theoretical_loss": 3.4356812879019176, + "tokens_seen": 1963458560 + }, + { + "epoch": 0.18, + "learning_rate": 0.000411979797979798, + "loss": 1.6616, + "theoretical_loss": 3.4356716920052897, + "tokens_seen": 1963524096 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041195959595959595, + "loss": 1.2516, + "theoretical_loss": 3.4356620965186107, + "tokens_seen": 1963589632 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041193939393939396, + "loss": 1.4148, + "theoretical_loss": 3.435652501441849, + "tokens_seen": 1963655168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119191919191919, + "loss": 1.4648, + "theoretical_loss": 3.4356429067749747, + "tokens_seen": 1963720704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118989898989899, + "loss": 1.4111, + "theoretical_loss": 3.4356333125179557, + "tokens_seen": 1963786240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118787878787879, + "loss": 1.4834, + "theoretical_loss": 3.435623718670761, + "tokens_seen": 1963851776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118585858585859, + "loss": 1.4329, + "theoretical_loss": 3.43561412523336, + "tokens_seen": 1963917312 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041183838383838386, + "loss": 1.4315, + "theoretical_loss": 3.4356045322057205, + "tokens_seen": 1963982848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118181818181818, + "loss": 1.1834, + "theoretical_loss": 3.435594939587812, + "tokens_seen": 1964048384 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041179797979797977, + "loss": 1.1604, + "theoretical_loss": 3.435585347379603, + "tokens_seen": 1964113920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117777777777778, + "loss": 1.3149, + "theoretical_loss": 3.4355757555810627, + "tokens_seen": 1964179456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117575757575758, + "loss": 1.4457, + "theoretical_loss": 3.435566164192159, + "tokens_seen": 1964244992 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041173737373737375, + "loss": 1.4566, + "theoretical_loss": 3.435556573212862, + "tokens_seen": 1964310528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117171717171717, + "loss": 1.4909, + "theoretical_loss": 3.4355469826431406, + "tokens_seen": 1964376064 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041169696969696966, + "loss": 1.3193, + "theoretical_loss": 3.435537392482962, + "tokens_seen": 1964441600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041167676767676773, + "loss": 1.5391, + "theoretical_loss": 3.4355278027322966, + "tokens_seen": 1964507136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116565656565657, + "loss": 1.4511, + "theoretical_loss": 3.435518213391113, + "tokens_seen": 1964572672 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041163636363636364, + "loss": 1.4846, + "theoretical_loss": 3.4355086244593793, + "tokens_seen": 1964638208 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.3200146555900574, + "objective/train/avg_token_score": 0.009661064483225346, + "objective/train/docs_used": 1110737, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0394518375396729, + "objective/train/original_loss": 3.6994714736938477, + "objective/train/theoretical_loss": 3.435499035937065, + "objective/train/tokens_used": 323562976, + "objective/train/unlikelihood_loss": 0.08720573782920837, + "theoretical_loss": 3.435499035937065, + "tokens_seen": 1964703744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116161616161616, + "loss": 1.2846, + "theoretical_loss": 3.435499035937065, + "tokens_seen": 1964703744 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041159595959595956, + "loss": 1.4118, + "theoretical_loss": 3.435489447824139, + "tokens_seen": 1964769280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115757575757576, + "loss": 1.3332, + "theoretical_loss": 3.43547986012057, + "tokens_seen": 1964834816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115555555555556, + "loss": 1.371, + "theoretical_loss": 3.4354702728263264, + "tokens_seen": 1964900352 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041153535353535354, + "loss": 1.5411, + "theoretical_loss": 3.435460685941378, + "tokens_seen": 1964965888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115151515151515, + "loss": 1.4533, + "theoretical_loss": 3.435451099465693, + "tokens_seen": 1965031424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114949494949495, + "loss": 1.2664, + "theoretical_loss": 3.4354415133992404, + "tokens_seen": 1965096960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114747474747475, + "loss": 1.4433, + "theoretical_loss": 3.4354319277419894, + "tokens_seen": 1965162496 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041145454545454547, + "loss": 1.4658, + "theoretical_loss": 3.4354223424939088, + "tokens_seen": 1965228032 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041143434343434343, + "loss": 1.382, + "theoretical_loss": 3.435412757654967, + "tokens_seen": 1965293568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114141414141414, + "loss": 1.3177, + "theoretical_loss": 3.4354031732251333, + "tokens_seen": 1965359104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004113939393939394, + "loss": 1.2478, + "theoretical_loss": 3.4353935892043768, + "tokens_seen": 1965424640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004113737373737374, + "loss": 1.7692, + "theoretical_loss": 3.435384005592666, + "tokens_seen": 1965490176 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041135353535353536, + "loss": 1.5251, + "theoretical_loss": 3.4353744223899696, + "tokens_seen": 1965555712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004113333333333333, + "loss": 1.441, + "theoretical_loss": 3.4353648395962573, + "tokens_seen": 1965621248 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041131313131313133, + "loss": 1.3334, + "theoretical_loss": 3.435355257211498, + "tokens_seen": 1965686784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112929292929293, + "loss": 1.4239, + "theoretical_loss": 3.4353456752356593, + "tokens_seen": 1965752320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112727272727273, + "loss": 1.3418, + "theoretical_loss": 3.4353360936687114, + "tokens_seen": 1965817856 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041125252525252526, + "loss": 1.1314, + "theoretical_loss": 3.4353265125106223, + "tokens_seen": 1965883392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112323232323232, + "loss": 1.4341, + "theoretical_loss": 3.4353169317613617, + "tokens_seen": 1965948928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112121212121212, + "loss": 1.5507, + "theoretical_loss": 3.4353073514208985, + "tokens_seen": 1966014464 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041119191919191924, + "loss": 1.3749, + "theoretical_loss": 3.4352977714892017, + "tokens_seen": 1966080000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111717171717172, + "loss": 1.5129, + "theoretical_loss": 3.4352881919662392, + "tokens_seen": 1966145536 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041115151515151515, + "loss": 1.382, + "theoretical_loss": 3.435278612851981, + "tokens_seen": 1966211072 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041113131313131316, + "loss": 1.6181, + "theoretical_loss": 3.435269034146396, + "tokens_seen": 1966276608 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.4528347849845886, + "objective/train/avg_token_score": 0.006985062267631292, + "objective/train/docs_used": 1112038, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5920250415802002, + "objective/train/original_loss": 4.1393818855285645, + "objective/train/theoretical_loss": 3.4352594558494527, + "objective/train/tokens_used": 325201376, + "objective/train/unlikelihood_loss": 0.0694524422287941, + "theoretical_loss": 3.4352594558494527, + "tokens_seen": 1966342144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111111111111111, + "loss": 1.3822, + "theoretical_loss": 3.4352594558494527, + "tokens_seen": 1966342144 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041109090909090913, + "loss": 1.7697, + "theoretical_loss": 3.4352498779611196, + "tokens_seen": 1966407680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110707070707071, + "loss": 1.4781, + "theoretical_loss": 3.435240300481367, + "tokens_seen": 1966473216 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041105050505050504, + "loss": 1.381, + "theoretical_loss": 3.4352307234101627, + "tokens_seen": 1966538752 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041103030303030305, + "loss": 1.4222, + "theoretical_loss": 3.4352211467474767, + "tokens_seen": 1966604288 + }, + { + "epoch": 0.19, + "learning_rate": 0.000411010101010101, + "loss": 1.5048, + "theoretical_loss": 3.4352115704932764, + "tokens_seen": 1966669824 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410989898989899, + "loss": 1.3564, + "theoretical_loss": 3.4352019946475325, + "tokens_seen": 1966735360 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410969696969697, + "loss": 1.1223, + "theoretical_loss": 3.435192419210213, + "tokens_seen": 1966800896 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410949494949495, + "loss": 1.5934, + "theoretical_loss": 3.435182844181287, + "tokens_seen": 1966866432 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041092929292929295, + "loss": 1.2154, + "theoretical_loss": 3.435173269560723, + "tokens_seen": 1966931968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109090909090909, + "loss": 1.4263, + "theoretical_loss": 3.435163695348491, + "tokens_seen": 1966997504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108888888888889, + "loss": 1.4043, + "theoretical_loss": 3.435154121544559, + "tokens_seen": 1967063040 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041086868686868687, + "loss": 1.263, + "theoretical_loss": 3.4351445481488967, + "tokens_seen": 1967128576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108484848484849, + "loss": 1.5556, + "theoretical_loss": 3.435134975161473, + "tokens_seen": 1967194112 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041082828282828284, + "loss": 1.6561, + "theoretical_loss": 3.4351254025822566, + "tokens_seen": 1967259648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108080808080808, + "loss": 1.3542, + "theoretical_loss": 3.435115830411217, + "tokens_seen": 1967325184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107878787878788, + "loss": 1.328, + "theoretical_loss": 3.4351062586483225, + "tokens_seen": 1967390720 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041076767676767676, + "loss": 1.6009, + "theoretical_loss": 3.4350966872935422, + "tokens_seen": 1967456256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107474747474748, + "loss": 1.3577, + "theoretical_loss": 3.4350871163468457, + "tokens_seen": 1967521792 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041072727272727273, + "loss": 1.3894, + "theoretical_loss": 3.435077545808202, + "tokens_seen": 1967587328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107070707070707, + "loss": 1.3641, + "theoretical_loss": 3.435067975677579, + "tokens_seen": 1967652864 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106868686868687, + "loss": 1.4913, + "theoretical_loss": 3.4350584059549467, + "tokens_seen": 1967718400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106666666666667, + "loss": 1.6145, + "theoretical_loss": 3.435048836640274, + "tokens_seen": 1967783936 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041064646464646467, + "loss": 1.5496, + "theoretical_loss": 3.4350392677335297, + "tokens_seen": 1967849472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106262626262626, + "loss": 1.4591, + "theoretical_loss": 3.435029699234683, + "tokens_seen": 1967915008 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.3913734257221222, + "objective/train/avg_token_score": 0.011882202699780464, + "objective/train/docs_used": 1112624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2495681047439575, + "objective/train/original_loss": 4.346351623535156, + "objective/train/theoretical_loss": 3.435020131143703, + "objective/train/tokens_used": 326839776, + "objective/train/unlikelihood_loss": 0.07031120359897614, + "theoretical_loss": 3.435020131143703, + "tokens_seen": 1967980544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106060606060606, + "loss": 1.2495, + "theoretical_loss": 3.435020131143703, + "tokens_seen": 1967980544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105858585858586, + "loss": 1.5182, + "theoretical_loss": 3.4350105634605583, + "tokens_seen": 1968046080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105656565656566, + "loss": 1.4065, + "theoretical_loss": 3.435000996185219, + "tokens_seen": 1968111616 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041054545454545456, + "loss": 1.4003, + "theoretical_loss": 3.4349914293176527, + "tokens_seen": 1968177152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105252525252525, + "loss": 1.4241, + "theoretical_loss": 3.4349818628578292, + "tokens_seen": 1968242688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105050505050505, + "loss": 1.5984, + "theoretical_loss": 3.4349722968057175, + "tokens_seen": 1968308224 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041048484848484854, + "loss": 1.6424, + "theoretical_loss": 3.4349627311612867, + "tokens_seen": 1968373760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104646464646465, + "loss": 1.5381, + "theoretical_loss": 3.434953165924506, + "tokens_seen": 1968439296 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041044444444444445, + "loss": 1.2385, + "theoretical_loss": 3.434943601095344, + "tokens_seen": 1968504832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104242424242424, + "loss": 1.2404, + "theoretical_loss": 3.43493403667377, + "tokens_seen": 1968570368 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041040404040404037, + "loss": 1.4081, + "theoretical_loss": 3.4349244726597528, + "tokens_seen": 1968635904 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041038383838383843, + "loss": 1.4257, + "theoretical_loss": 3.434914909053262, + "tokens_seen": 1968701440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103636363636364, + "loss": 1.2265, + "theoretical_loss": 3.434905345854266, + "tokens_seen": 1968766976 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041034343434343435, + "loss": 1.258, + "theoretical_loss": 3.4348957830627347, + "tokens_seen": 1968832512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103232323232323, + "loss": 1.4968, + "theoretical_loss": 3.434886220678637, + "tokens_seen": 1968898048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103030303030303, + "loss": 1.4694, + "theoretical_loss": 3.434876658701941, + "tokens_seen": 1968963584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102828282828283, + "loss": 1.5733, + "theoretical_loss": 3.434867097132617, + "tokens_seen": 1969029120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102626262626263, + "loss": 1.5411, + "theoretical_loss": 3.4348575359706333, + "tokens_seen": 1969094656 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041024242424242424, + "loss": 1.4478, + "theoretical_loss": 3.43484797521596, + "tokens_seen": 1969160192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102222222222222, + "loss": 1.3074, + "theoretical_loss": 3.4348384148685644, + "tokens_seen": 1969225728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102020202020202, + "loss": 1.2974, + "theoretical_loss": 3.4348288549284174, + "tokens_seen": 1969291264 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101818181818182, + "loss": 1.3546, + "theoretical_loss": 3.434819295395487, + "tokens_seen": 1969356800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101616161616162, + "loss": 1.1625, + "theoretical_loss": 3.434809736269743, + "tokens_seen": 1969422336 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041014141414141413, + "loss": 1.5824, + "theoretical_loss": 3.4348001775511543, + "tokens_seen": 1969487872 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041012121212121214, + "loss": 1.3634, + "theoretical_loss": 3.4347906192396893, + "tokens_seen": 1969553408 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.27834799885749817, + "objective/train/avg_token_score": 0.007226386573165655, + "objective/train/docs_used": 1113989, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9873694777488708, + "objective/train/original_loss": 4.861571788787842, + "objective/train/theoretical_loss": 3.4347810613353182, + "objective/train/tokens_used": 328478176, + "objective/train/unlikelihood_loss": 0.06607561558485031, + "theoretical_loss": 3.4347810613353182, + "tokens_seen": 1969618944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101010101010101, + "loss": 1.2623, + "theoretical_loss": 3.4347810613353182, + "tokens_seen": 1969618944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100808080808081, + "loss": 1.5619, + "theoretical_loss": 3.434771503838009, + "tokens_seen": 1969684480 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041006060606060607, + "loss": 1.1316, + "theoretical_loss": 3.434761946747732, + "tokens_seen": 1969750016 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410040404040404, + "loss": 1.38, + "theoretical_loss": 3.434752390064456, + "tokens_seen": 1969815552 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041002020202020203, + "loss": 1.4082, + "theoretical_loss": 3.4347428337881496, + "tokens_seen": 1969881088 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041, + "loss": 1.2224, + "theoretical_loss": 3.434733277918782, + "tokens_seen": 1969946624 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409979797979798, + "loss": 1.3039, + "theoretical_loss": 3.4347237224563223, + "tokens_seen": 1970012160 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040995959595959596, + "loss": 1.3388, + "theoretical_loss": 3.4347141674007404, + "tokens_seen": 1970077696 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040993939393939397, + "loss": 1.3502, + "theoretical_loss": 3.4347046127520047, + "tokens_seen": 1970143232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099191919191919, + "loss": 1.3016, + "theoretical_loss": 3.4346950585100853, + "tokens_seen": 1970208768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098989898989899, + "loss": 1.3919, + "theoretical_loss": 3.43468550467495, + "tokens_seen": 1970274304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098787878787879, + "loss": 1.5056, + "theoretical_loss": 3.434675951246568, + "tokens_seen": 1970339840 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040985858585858585, + "loss": 1.2892, + "theoretical_loss": 3.43466639822491, + "tokens_seen": 1970405376 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040983838383838386, + "loss": 1.4575, + "theoretical_loss": 3.434656845609944, + "tokens_seen": 1970470912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098181818181818, + "loss": 1.4339, + "theoretical_loss": 3.434647293401639, + "tokens_seen": 1970536448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097979797979798, + "loss": 1.1932, + "theoretical_loss": 3.434637741599965, + "tokens_seen": 1970601984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097777777777778, + "loss": 1.4734, + "theoretical_loss": 3.4346281902048905, + "tokens_seen": 1970667520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097575757575758, + "loss": 1.4094, + "theoretical_loss": 3.4346186392163847, + "tokens_seen": 1970733056 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040973737373737376, + "loss": 1.5061, + "theoretical_loss": 3.434609088634417, + "tokens_seen": 1970798592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097171717171717, + "loss": 1.7504, + "theoretical_loss": 3.4345995384589565, + "tokens_seen": 1970864128 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040969696969696967, + "loss": 1.6252, + "theoretical_loss": 3.4345899886899725, + "tokens_seen": 1970929664 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096767676767677, + "loss": 1.6023, + "theoretical_loss": 3.434580439327434, + "tokens_seen": 1970995200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096565656565657, + "loss": 1.4679, + "theoretical_loss": 3.4345708903713104, + "tokens_seen": 1971060736 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040963636363636365, + "loss": 1.3235, + "theoretical_loss": 3.4345613418215706, + "tokens_seen": 1971126272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096161616161616, + "loss": 1.1655, + "theoretical_loss": 3.4345517936781835, + "tokens_seen": 1971191808 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.5766128897666931, + "objective/train/avg_token_score": 0.006864483468234539, + "objective/train/docs_used": 1114737, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9049957990646362, + "objective/train/original_loss": 4.060426235198975, + "objective/train/theoretical_loss": 3.4345422459411195, + "objective/train/tokens_used": 330116576, + "objective/train/unlikelihood_loss": 0.039708707481622696, + "theoretical_loss": 3.4345422459411195, + "tokens_seen": 1971257344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040959595959595956, + "loss": 1.6513, + "theoretical_loss": 3.4345422459411195, + "tokens_seen": 1971257344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040957575757575763, + "loss": 1.5602, + "theoretical_loss": 3.434532698610347, + "tokens_seen": 1971322880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095555555555556, + "loss": 1.3076, + "theoretical_loss": 3.434523151685835, + "tokens_seen": 1971388416 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040953535353535354, + "loss": 1.1315, + "theoretical_loss": 3.4345136051675524, + "tokens_seen": 1971453952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095151515151515, + "loss": 1.3409, + "theoretical_loss": 3.43450405905547, + "tokens_seen": 1971519488 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040949494949494945, + "loss": 1.1566, + "theoretical_loss": 3.4344945133495552, + "tokens_seen": 1971585024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094747474747475, + "loss": 1.4211, + "theoretical_loss": 3.4344849680497784, + "tokens_seen": 1971650560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094545454545455, + "loss": 1.323, + "theoretical_loss": 3.4344754231561083, + "tokens_seen": 1971716096 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040943434343434343, + "loss": 1.5279, + "theoretical_loss": 3.4344658786685143, + "tokens_seen": 1971781632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094141414141414, + "loss": 1.3783, + "theoretical_loss": 3.4344563345869656, + "tokens_seen": 1971847168 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040939393939393946, + "loss": 1.3743, + "theoretical_loss": 3.4344467909114313, + "tokens_seen": 1971912704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093737373737374, + "loss": 1.3636, + "theoretical_loss": 3.4344372476418807, + "tokens_seen": 1971978240 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040935353535353537, + "loss": 1.3911, + "theoretical_loss": 3.434427704778283, + "tokens_seen": 1972043776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093333333333333, + "loss": 1.4582, + "theoretical_loss": 3.434418162320608, + "tokens_seen": 1972109312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093131313131313, + "loss": 1.1357, + "theoretical_loss": 3.434408620268824, + "tokens_seen": 1972174848 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040929292929292935, + "loss": 1.5361, + "theoretical_loss": 3.434399078622901, + "tokens_seen": 1972240384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092727272727273, + "loss": 1.7598, + "theoretical_loss": 3.434389537382808, + "tokens_seen": 1972305920 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040925252525252526, + "loss": 1.365, + "theoretical_loss": 3.434379996548514, + "tokens_seen": 1972371456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092323232323232, + "loss": 1.533, + "theoretical_loss": 3.4343704561199884, + "tokens_seen": 1972436992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092121212121212, + "loss": 1.4524, + "theoretical_loss": 3.4343609160972006, + "tokens_seen": 1972502528 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040919191919191924, + "loss": 1.4198, + "theoretical_loss": 3.4343513764801203, + "tokens_seen": 1972568064 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091717171717172, + "loss": 1.3793, + "theoretical_loss": 3.4343418372687156, + "tokens_seen": 1972633600 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040915151515151515, + "loss": 1.3158, + "theoretical_loss": 3.434332298462957, + "tokens_seen": 1972699136 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091313131313131, + "loss": 1.382, + "theoretical_loss": 3.434322760062813, + "tokens_seen": 1972764672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091111111111111, + "loss": 1.3222, + "theoretical_loss": 3.434313222068253, + "tokens_seen": 1972830208 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.38007086515426636, + "objective/train/avg_token_score": 0.014573216438293457, + "objective/train/docs_used": 1116152, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2298966646194458, + "objective/train/original_loss": 4.61524772644043, + "objective/train/theoretical_loss": 3.4343036844792465, + "objective/train/tokens_used": 331754976, + "objective/train/unlikelihood_loss": 0.04789040982723236, + "theoretical_loss": 3.4343036844792465, + "tokens_seen": 1972895744 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040909090909090913, + "loss": 1.2967, + "theoretical_loss": 3.4343036844792465, + "tokens_seen": 1972895744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090707070707071, + "loss": 1.4073, + "theoretical_loss": 3.4342941472957627, + "tokens_seen": 1972961280 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040905050505050505, + "loss": 1.4403, + "theoretical_loss": 3.4342846105177705, + "tokens_seen": 1973026816 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409030303030303, + "loss": 1.3333, + "theoretical_loss": 3.4342750741452397, + "tokens_seen": 1973092352 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409010101010101, + "loss": 1.5288, + "theoretical_loss": 3.43426553817814, + "tokens_seen": 1973157888 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408989898989899, + "loss": 1.3896, + "theoretical_loss": 3.4342560026164395, + "tokens_seen": 1973223424 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408969696969697, + "loss": 1.3534, + "theoretical_loss": 3.4342464674601083, + "tokens_seen": 1973288960 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040894949494949494, + "loss": 1.3614, + "theoretical_loss": 3.4342369327091156, + "tokens_seen": 1973354496 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040892929292929295, + "loss": 1.6497, + "theoretical_loss": 3.434227398363431, + "tokens_seen": 1973420032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089090909090909, + "loss": 1.577, + "theoretical_loss": 3.4342178644230232, + "tokens_seen": 1973485568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088888888888889, + "loss": 1.2795, + "theoretical_loss": 3.4342083308878615, + "tokens_seen": 1973551104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088686868686869, + "loss": 1.4452, + "theoretical_loss": 3.4341987977579156, + "tokens_seen": 1973616640 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040884848484848483, + "loss": 1.4619, + "theoretical_loss": 3.434189265033155, + "tokens_seen": 1973682176 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040882828282828284, + "loss": 1.4424, + "theoretical_loss": 3.4341797327135484, + "tokens_seen": 1973747712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088080808080808, + "loss": 1.1546, + "theoretical_loss": 3.4341702007990658, + "tokens_seen": 1973813248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087878787878788, + "loss": 1.4294, + "theoretical_loss": 3.434160669289676, + "tokens_seen": 1973878784 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040876767676767677, + "loss": 0.9521, + "theoretical_loss": 3.4341511381853485, + "tokens_seen": 1973944320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087474747474748, + "loss": 1.5766, + "theoretical_loss": 3.434141607486053, + "tokens_seen": 1974009856 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040872727272727274, + "loss": 1.4657, + "theoretical_loss": 3.4341320771917583, + "tokens_seen": 1974075392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087070707070707, + "loss": 1.4426, + "theoretical_loss": 3.434122547302434, + "tokens_seen": 1974140928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086868686868687, + "loss": 1.5492, + "theoretical_loss": 3.4341130178180492, + "tokens_seen": 1974206464 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040866666666666666, + "loss": 1.3706, + "theoretical_loss": 3.4341034887385735, + "tokens_seen": 1974272000 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040864646464646467, + "loss": 1.5491, + "theoretical_loss": 3.4340939600639766, + "tokens_seen": 1974337536 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040862626262626263, + "loss": 1.3823, + "theoretical_loss": 3.4340844317942274, + "tokens_seen": 1974403072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086060606060606, + "loss": 1.3105, + "theoretical_loss": 3.4340749039292957, + "tokens_seen": 1974468608 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.46144917607307434, + "objective/train/avg_token_score": 0.03224779665470123, + "objective/train/docs_used": 1116806, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.653957486152649, + "objective/train/original_loss": 4.292056560516357, + "objective/train/theoretical_loss": 3.4340653764691496, + "objective/train/tokens_used": 333393376, + "objective/train/unlikelihood_loss": 0.05980272591114044, + "theoretical_loss": 3.4340653764691496, + "tokens_seen": 1974534144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085858585858586, + "loss": 1.4107, + "theoretical_loss": 3.4340653764691496, + "tokens_seen": 1974534144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085656565656566, + "loss": 1.4226, + "theoretical_loss": 3.43405584941376, + "tokens_seen": 1974599680 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040854545454545457, + "loss": 1.3297, + "theoretical_loss": 3.4340463227630957, + "tokens_seen": 1974665216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085252525252525, + "loss": 1.4029, + "theoretical_loss": 3.434036796517126, + "tokens_seen": 1974730752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085050505050505, + "loss": 1.3152, + "theoretical_loss": 3.43402727067582, + "tokens_seen": 1974796288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084848484848485, + "loss": 1.6887, + "theoretical_loss": 3.4340177452391476, + "tokens_seen": 1974861824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084646464646465, + "loss": 1.3779, + "theoretical_loss": 3.4340082202070774, + "tokens_seen": 1974927360 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040844444444444446, + "loss": 1.3655, + "theoretical_loss": 3.43399869557958, + "tokens_seen": 1974992896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084242424242424, + "loss": 1.5568, + "theoretical_loss": 3.4339891713566244, + "tokens_seen": 1975058432 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040840404040404037, + "loss": 1.4729, + "theoretical_loss": 3.433979647538179, + "tokens_seen": 1975123968 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040838383838383844, + "loss": 1.3622, + "theoretical_loss": 3.4339701241242144, + "tokens_seen": 1975189504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083636363636364, + "loss": 1.2509, + "theoretical_loss": 3.4339606011146993, + "tokens_seen": 1975255040 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040834343434343435, + "loss": 1.2185, + "theoretical_loss": 3.4339510785096037, + "tokens_seen": 1975320576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083232323232323, + "loss": 1.4309, + "theoretical_loss": 3.4339415563088966, + "tokens_seen": 1975386112 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040830303030303026, + "loss": 1.4844, + "theoretical_loss": 3.433932034512547, + "tokens_seen": 1975451648 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040828282828282833, + "loss": 1.2712, + "theoretical_loss": 3.4339225131205255, + "tokens_seen": 1975517184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082626262626263, + "loss": 1.4414, + "theoretical_loss": 3.4339129921328, + "tokens_seen": 1975582720 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040824242424242424, + "loss": 1.5263, + "theoretical_loss": 3.4339034715493417, + "tokens_seen": 1975648256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082222222222222, + "loss": 1.1537, + "theoretical_loss": 3.433893951370118, + "tokens_seen": 1975713792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082020202020202, + "loss": 1.4704, + "theoretical_loss": 3.4338844315951, + "tokens_seen": 1975779328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081818181818182, + "loss": 1.41, + "theoretical_loss": 3.4338749122242564, + "tokens_seen": 1975844864 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081616161616162, + "loss": 1.5023, + "theoretical_loss": 3.433865393257557, + "tokens_seen": 1975910400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040814141414141414, + "loss": 1.5095, + "theoretical_loss": 3.433855874694971, + "tokens_seen": 1975975936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081212121212121, + "loss": 1.5097, + "theoretical_loss": 3.433846356536468, + "tokens_seen": 1976041472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081010101010101, + "loss": 1.1897, + "theoretical_loss": 3.4338368387820166, + "tokens_seen": 1976107008 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.38624143600463867, + "objective/train/avg_token_score": 0.0034812039230018854, + "objective/train/docs_used": 1117380, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2491487264633179, + "objective/train/original_loss": 4.108737945556641, + "objective/train/theoretical_loss": 3.433827321431587, + "objective/train/tokens_used": 335031776, + "objective/train/unlikelihood_loss": 0.0697765201330185, + "theoretical_loss": 3.433827321431587, + "tokens_seen": 1976172544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004080808080808081, + "loss": 1.4446, + "theoretical_loss": 3.433827321431587, + "tokens_seen": 1976172544 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040806060606060607, + "loss": 1.293, + "theoretical_loss": 3.433817804485149, + "tokens_seen": 1976238080 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040804040404040403, + "loss": 1.415, + "theoretical_loss": 3.4338082879426715, + "tokens_seen": 1976303616 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040802020202020204, + "loss": 1.3463, + "theoretical_loss": 3.433798771804124, + "tokens_seen": 1976369152 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408, + "loss": 1.6793, + "theoretical_loss": 3.433789256069476, + "tokens_seen": 1976434688 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407979797979798, + "loss": 1.3098, + "theoretical_loss": 3.433779740738697, + "tokens_seen": 1976500224 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040795959595959596, + "loss": 1.4687, + "theoretical_loss": 3.433770225811757, + "tokens_seen": 1976565760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079393939393939, + "loss": 1.6195, + "theoretical_loss": 3.433760711288624, + "tokens_seen": 1976631296 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040791919191919193, + "loss": 1.4593, + "theoretical_loss": 3.4337511971692694, + "tokens_seen": 1976696832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078989898989899, + "loss": 1.4784, + "theoretical_loss": 3.433741683453661, + "tokens_seen": 1976762368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078787878787879, + "loss": 1.4241, + "theoretical_loss": 3.4337321701417696, + "tokens_seen": 1976827904 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040785858585858586, + "loss": 1.6483, + "theoretical_loss": 3.433722657233564, + "tokens_seen": 1976893440 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040783838383838387, + "loss": 1.3813, + "theoretical_loss": 3.4337131447290132, + "tokens_seen": 1976958976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078181818181818, + "loss": 1.5208, + "theoretical_loss": 3.4337036326280876, + "tokens_seen": 1977024512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077979797979798, + "loss": 1.3666, + "theoretical_loss": 3.433694120930756, + "tokens_seen": 1977090048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077777777777778, + "loss": 1.5639, + "theoretical_loss": 3.4336846096369893, + "tokens_seen": 1977155584 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040775757575757575, + "loss": 1.3248, + "theoretical_loss": 3.433675098746755, + "tokens_seen": 1977221120 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040773737373737376, + "loss": 1.2001, + "theoretical_loss": 3.4336655882600238, + "tokens_seen": 1977286656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077171717171717, + "loss": 1.3965, + "theoretical_loss": 3.433656078176765, + "tokens_seen": 1977352192 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040769696969696973, + "loss": 1.3558, + "theoretical_loss": 3.4336465684969477, + "tokens_seen": 1977417728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076767676767677, + "loss": 1.4859, + "theoretical_loss": 3.4336370592205423, + "tokens_seen": 1977483264 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076565656565657, + "loss": 1.3785, + "theoretical_loss": 3.4336275503475178, + "tokens_seen": 1977548800 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040763636363636365, + "loss": 1.6469, + "theoretical_loss": 3.4336180418778435, + "tokens_seen": 1977614336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076161616161616, + "loss": 1.3878, + "theoretical_loss": 3.4336085338114897, + "tokens_seen": 1977679872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075959595959596, + "loss": 1.2246, + "theoretical_loss": 3.433599026148425, + "tokens_seen": 1977745408 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.2834188640117645, + "objective/train/avg_token_score": 0.007966935634613037, + "objective/train/docs_used": 1118619, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9379705190658569, + "objective/train/original_loss": 4.463184356689453, + "objective/train/theoretical_loss": 3.4335895188886187, + "objective/train/tokens_used": 336670176, + "objective/train/unlikelihood_loss": 0.07986027747392654, + "theoretical_loss": 3.4335895188886187, + "tokens_seen": 1977810944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075757575757576, + "loss": 1.4292, + "theoretical_loss": 3.4335895188886187, + "tokens_seen": 1977810944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075555555555556, + "loss": 1.4095, + "theoretical_loss": 3.433580012032042, + "tokens_seen": 1977876480 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040753535353535355, + "loss": 1.4814, + "theoretical_loss": 3.433570505578663, + "tokens_seen": 1977942016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075151515151515, + "loss": 1.5789, + "theoretical_loss": 3.433560999528451, + "tokens_seen": 1978007552 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074949494949495, + "loss": 1.5362, + "theoretical_loss": 3.433551493881377, + "tokens_seen": 1978073088 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040747474747474747, + "loss": 1.3804, + "theoretical_loss": 3.4335419886374092, + "tokens_seen": 1978138624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074545454545455, + "loss": 1.6188, + "theoretical_loss": 3.433532483796518, + "tokens_seen": 1978204160 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040743434343434344, + "loss": 1.1727, + "theoretical_loss": 3.433522979358673, + "tokens_seen": 1978269696 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074141414141414, + "loss": 1.5376, + "theoretical_loss": 3.433513475323843, + "tokens_seen": 1978335232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073939393939394, + "loss": 1.2815, + "theoretical_loss": 3.433503971691998, + "tokens_seen": 1978400768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073737373737374, + "loss": 1.4291, + "theoretical_loss": 3.433494468463107, + "tokens_seen": 1978466304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073535353535354, + "loss": 1.3798, + "theoretical_loss": 3.4334849656371405, + "tokens_seen": 1978531840 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040733333333333333, + "loss": 1.3716, + "theoretical_loss": 3.4334754632140676, + "tokens_seen": 1978597376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073131313131313, + "loss": 1.327, + "theoretical_loss": 3.433465961193858, + "tokens_seen": 1978662912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072929292929293, + "loss": 1.3915, + "theoretical_loss": 3.4334564595764814, + "tokens_seen": 1978728448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072727272727273, + "loss": 1.543, + "theoretical_loss": 3.4334469583619067, + "tokens_seen": 1978793984 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040725252525252527, + "loss": 1.3711, + "theoretical_loss": 3.433437457550104, + "tokens_seen": 1978859520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072323232323232, + "loss": 1.2373, + "theoretical_loss": 3.4334279571410438, + "tokens_seen": 1978925056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072121212121212, + "loss": 1.4841, + "theoretical_loss": 3.4334184571346933, + "tokens_seen": 1978990592 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040719191919191925, + "loss": 1.5336, + "theoretical_loss": 3.4334089575310247, + "tokens_seen": 1979056128 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071717171717172, + "loss": 1.4586, + "theoretical_loss": 3.433399458330006, + "tokens_seen": 1979121664 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040715151515151516, + "loss": 1.2878, + "theoretical_loss": 3.433389959531607, + "tokens_seen": 1979187200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071313131313131, + "loss": 1.5874, + "theoretical_loss": 3.433380461135798, + "tokens_seen": 1979252736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071111111111111, + "loss": 1.5125, + "theoretical_loss": 3.433370963142548, + "tokens_seen": 1979318272 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040709090909090914, + "loss": 1.7091, + "theoretical_loss": 3.433361465551827, + "tokens_seen": 1979383808 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.34341397881507874, + "objective/train/avg_token_score": 0.009266973473131657, + "objective/train/docs_used": 1119354, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2784351110458374, + "objective/train/original_loss": 4.967345714569092, + "objective/train/theoretical_loss": 3.4333519683636036, + "objective/train/tokens_used": 338308576, + "objective/train/unlikelihood_loss": 0.047723278403282166, + "theoretical_loss": 3.4333519683636036, + "tokens_seen": 1979449344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070707070707071, + "loss": 1.5983, + "theoretical_loss": 3.4333519683636036, + "tokens_seen": 1979449344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040705050505050505, + "loss": 1.3467, + "theoretical_loss": 3.433342471577849, + "tokens_seen": 1979514880 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407030303030303, + "loss": 1.4703, + "theoretical_loss": 3.4333329751945314, + "tokens_seen": 1979580416 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407010101010101, + "loss": 1.607, + "theoretical_loss": 3.433323479213622, + "tokens_seen": 1979645952 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040698989898989903, + "loss": 1.3784, + "theoretical_loss": 3.4333139836350886, + "tokens_seen": 1979711488 + }, + { + "epoch": 0.19, + "learning_rate": 0.000406969696969697, + "loss": 1.3995, + "theoretical_loss": 3.433304488458902, + "tokens_seen": 1979777024 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040694949494949495, + "loss": 1.6541, + "theoretical_loss": 3.433294993685031, + "tokens_seen": 1979842560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069292929292929, + "loss": 1.2676, + "theoretical_loss": 3.4332854993134463, + "tokens_seen": 1979908096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069090909090909, + "loss": 1.3127, + "theoretical_loss": 3.4332760053441174, + "tokens_seen": 1979973632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068888888888889, + "loss": 1.4902, + "theoretical_loss": 3.4332665117770125, + "tokens_seen": 1980039168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068686868686869, + "loss": 1.1425, + "theoretical_loss": 3.433257018612103, + "tokens_seen": 1980104704 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040684848484848484, + "loss": 1.5914, + "theoretical_loss": 3.433247525849358, + "tokens_seen": 1980170240 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040682828282828285, + "loss": 1.4876, + "theoretical_loss": 3.433238033488747, + "tokens_seen": 1980235776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068080808080808, + "loss": 1.5058, + "theoretical_loss": 3.433228541530239, + "tokens_seen": 1980301312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067878787878788, + "loss": 1.3863, + "theoretical_loss": 3.433219049973805, + "tokens_seen": 1980366848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067676767676768, + "loss": 1.5363, + "theoretical_loss": 3.4332095588194136, + "tokens_seen": 1980432384 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040674747474747473, + "loss": 1.1681, + "theoretical_loss": 3.433200068067035, + "tokens_seen": 1980497920 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040672727272727274, + "loss": 1.3775, + "theoretical_loss": 3.4331905777166387, + "tokens_seen": 1980563456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067070707070707, + "loss": 1.5538, + "theoretical_loss": 3.4331810877681948, + "tokens_seen": 1980628992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066868686868687, + "loss": 1.3664, + "theoretical_loss": 3.433171598221672, + "tokens_seen": 1980694528 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040666666666666667, + "loss": 1.2929, + "theoretical_loss": 3.43316210907704, + "tokens_seen": 1980760064 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066464646464647, + "loss": 1.6327, + "theoretical_loss": 3.43315262033427, + "tokens_seen": 1980825600 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040662626262626264, + "loss": 1.3788, + "theoretical_loss": 3.4331431319933303, + "tokens_seen": 1980891136 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066060606060606, + "loss": 1.3966, + "theoretical_loss": 3.4331336440541915, + "tokens_seen": 1980956672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004065858585858586, + "loss": 1.5779, + "theoretical_loss": 3.433124156516823, + "tokens_seen": 1981022208 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.41037389636039734, + "objective/train/avg_token_score": 0.023919859901070595, + "objective/train/docs_used": 1120511, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3339953422546387, + "objective/train/original_loss": 4.790691375732422, + "objective/train/theoretical_loss": 3.4331146693811934, + "objective/train/tokens_used": 339946976, + "objective/train/unlikelihood_loss": 0.049351904541254044, + "theoretical_loss": 3.4331146693811934, + "tokens_seen": 1981087744 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040656565656565656, + "loss": 1.422, + "theoretical_loss": 3.4331146693811934, + "tokens_seen": 1981087744 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040654545454545457, + "loss": 1.6359, + "theoretical_loss": 3.4331051826472736, + "tokens_seen": 1981153280 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040652525252525253, + "loss": 1.3955, + "theoretical_loss": 3.433095696315033, + "tokens_seen": 1981218816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004065050505050505, + "loss": 1.5511, + "theoretical_loss": 3.4330862103844417, + "tokens_seen": 1981284352 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064848484848485, + "loss": 1.4746, + "theoretical_loss": 3.4330767248554688, + "tokens_seen": 1981349888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064646464646465, + "loss": 1.6425, + "theoretical_loss": 3.433067239728084, + "tokens_seen": 1981415424 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040644444444444446, + "loss": 1.1774, + "theoretical_loss": 3.4330577550022574, + "tokens_seen": 1981480960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064242424242424, + "loss": 1.5055, + "theoretical_loss": 3.4330482706779586, + "tokens_seen": 1981546496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064040404040404, + "loss": 1.3855, + "theoretical_loss": 3.4330387867551573, + "tokens_seen": 1981612032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063838383838384, + "loss": 1.3515, + "theoretical_loss": 3.4330293032338233, + "tokens_seen": 1981677568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063636363636364, + "loss": 1.4944, + "theoretical_loss": 3.433019820113926, + "tokens_seen": 1981743104 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040634343434343436, + "loss": 1.4746, + "theoretical_loss": 3.4330103373954355, + "tokens_seen": 1981808640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063232323232323, + "loss": 1.2959, + "theoretical_loss": 3.4330008550783213, + "tokens_seen": 1981874176 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040630303030303027, + "loss": 1.5334, + "theoretical_loss": 3.4329913731625536, + "tokens_seen": 1981939712 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040628282828282834, + "loss": 1.2631, + "theoretical_loss": 3.4329818916481014, + "tokens_seen": 1982005248 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062626262626263, + "loss": 1.6203, + "theoretical_loss": 3.4329724105349353, + "tokens_seen": 1982070784 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040624242424242425, + "loss": 1.4313, + "theoretical_loss": 3.432962929823024, + "tokens_seen": 1982136320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062222222222222, + "loss": 1.415, + "theoretical_loss": 3.432953449512338, + "tokens_seen": 1982201856 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040620202020202016, + "loss": 1.4721, + "theoretical_loss": 3.432943969602847, + "tokens_seen": 1982267392 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040618181818181823, + "loss": 1.4335, + "theoretical_loss": 3.432934490094521, + "tokens_seen": 1982332928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061616161616162, + "loss": 1.4509, + "theoretical_loss": 3.4329250109873293, + "tokens_seen": 1982398464 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040614141414141414, + "loss": 1.4384, + "theoretical_loss": 3.4329155322812417, + "tokens_seen": 1982464000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061212121212121, + "loss": 1.4729, + "theoretical_loss": 3.432906053976228, + "tokens_seen": 1982529536 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040610101010101006, + "loss": 1.4978, + "theoretical_loss": 3.432896576072258, + "tokens_seen": 1982595072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060808080808081, + "loss": 1.5901, + "theoretical_loss": 3.432887098569301, + "tokens_seen": 1982660608 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.5727028250694275, + "objective/train/avg_token_score": 0.004548849072307348, + "objective/train/docs_used": 1121670, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.048727512359619, + "objective/train/original_loss": 4.066925525665283, + "objective/train/theoretical_loss": 3.4328776214673278, + "objective/train/tokens_used": 341585376, + "objective/train/unlikelihood_loss": 0.07542819529771805, + "theoretical_loss": 3.4328776214673278, + "tokens_seen": 1982726144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060606060606061, + "loss": 1.7648, + "theoretical_loss": 3.4328776214673278, + "tokens_seen": 1982726144 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040604040404040403, + "loss": 1.4492, + "theoretical_loss": 3.432868144766308, + "tokens_seen": 1982791680 + }, + { + "epoch": 0.2, + "learning_rate": 0.000406020202020202, + "loss": 1.4899, + "theoretical_loss": 3.432858668466211, + "tokens_seen": 1982857216 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040600000000000006, + "loss": 1.5625, + "theoretical_loss": 3.432849192567006, + "tokens_seen": 1982922752 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405979797979798, + "loss": 1.49, + "theoretical_loss": 3.4328397170686635, + "tokens_seen": 1982988288 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040595959595959597, + "loss": 1.6991, + "theoretical_loss": 3.4328302419711534, + "tokens_seen": 1983053824 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040593939393939393, + "loss": 1.5363, + "theoretical_loss": 3.4328207672744453, + "tokens_seen": 1983119360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004059191919191919, + "loss": 1.3888, + "theoretical_loss": 3.432811292978509, + "tokens_seen": 1983184896 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040589898989898995, + "loss": 1.4623, + "theoretical_loss": 3.432801819083315, + "tokens_seen": 1983250432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058787878787879, + "loss": 1.2032, + "theoretical_loss": 3.4327923455888314, + "tokens_seen": 1983315968 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040585858585858586, + "loss": 1.5196, + "theoretical_loss": 3.4327828724950296, + "tokens_seen": 1983381504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058383838383838, + "loss": 1.3848, + "theoretical_loss": 3.432773399801879, + "tokens_seen": 1983447040 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040581818181818183, + "loss": 1.2752, + "theoretical_loss": 3.432763927509349, + "tokens_seen": 1983512576 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040579797979797984, + "loss": 1.4994, + "theoretical_loss": 3.4327544556174097, + "tokens_seen": 1983578112 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057777777777778, + "loss": 1.5548, + "theoretical_loss": 3.432744984126031, + "tokens_seen": 1983643648 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040575757575757576, + "loss": 1.3647, + "theoretical_loss": 3.4327355130351824, + "tokens_seen": 1983709184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057373737373737, + "loss": 1.3819, + "theoretical_loss": 3.4327260423448345, + "tokens_seen": 1983774720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057171717171717, + "loss": 1.1727, + "theoretical_loss": 3.4327165720549564, + "tokens_seen": 1983840256 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040569696969696973, + "loss": 1.442, + "theoretical_loss": 3.4327071021655176, + "tokens_seen": 1983905792 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056767676767677, + "loss": 1.3211, + "theoretical_loss": 3.432697632676489, + "tokens_seen": 1983971328 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040565656565656565, + "loss": 1.4399, + "theoretical_loss": 3.43268816358784, + "tokens_seen": 1984036864 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040563636363636366, + "loss": 1.401, + "theoretical_loss": 3.4326786948995407, + "tokens_seen": 1984102400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056161616161616, + "loss": 1.7314, + "theoretical_loss": 3.43266922661156, + "tokens_seen": 1984167936 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040559595959595963, + "loss": 1.4885, + "theoretical_loss": 3.4326597587238687, + "tokens_seen": 1984233472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055757575757576, + "loss": 1.193, + "theoretical_loss": 3.432650291236436, + "tokens_seen": 1984299008 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.46981915831565857, + "objective/train/avg_token_score": 0.007897254079580307, + "objective/train/docs_used": 1122413, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6054514646530151, + "objective/train/original_loss": 4.072227954864502, + "objective/train/theoretical_loss": 3.4326408241492326, + "objective/train/tokens_used": 343223776, + "objective/train/unlikelihood_loss": 0.05646788701415062, + "theoretical_loss": 3.4326408241492326, + "tokens_seen": 1984364544 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040555555555555554, + "loss": 1.4734, + "theoretical_loss": 3.4326408241492326, + "tokens_seen": 1984364544 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040553535353535355, + "loss": 1.5419, + "theoretical_loss": 3.4326313574622276, + "tokens_seen": 1984430080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055151515151515, + "loss": 1.4651, + "theoretical_loss": 3.432621891175391, + "tokens_seen": 1984495616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054949494949495, + "loss": 1.5656, + "theoretical_loss": 3.432612425288693, + "tokens_seen": 1984561152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054747474747475, + "loss": 1.3567, + "theoretical_loss": 3.4326029598021033, + "tokens_seen": 1984626688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054545454545455, + "loss": 1.5375, + "theoretical_loss": 3.432593494715592, + "tokens_seen": 1984692224 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040543434343434344, + "loss": 1.2787, + "theoretical_loss": 3.432584030029128, + "tokens_seen": 1984757760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054141414141414, + "loss": 1.5412, + "theoretical_loss": 3.4325745657426827, + "tokens_seen": 1984823296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053939393939394, + "loss": 1.6001, + "theoretical_loss": 3.432565101856224, + "tokens_seen": 1984888832 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040537373737373737, + "loss": 1.4601, + "theoretical_loss": 3.4325556383697244, + "tokens_seen": 1984954368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053535353535354, + "loss": 1.4659, + "theoretical_loss": 3.432546175283152, + "tokens_seen": 1985019904 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040533333333333334, + "loss": 1.4451, + "theoretical_loss": 3.4325367125964767, + "tokens_seen": 1985085440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053131313131313, + "loss": 1.476, + "theoretical_loss": 3.4325272503096693, + "tokens_seen": 1985150976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052929292929293, + "loss": 1.4559, + "theoretical_loss": 3.4325177884226985, + "tokens_seen": 1985216512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052727272727273, + "loss": 1.4112, + "theoretical_loss": 3.4325083269355354, + "tokens_seen": 1985282048 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052525252525253, + "loss": 1.6486, + "theoretical_loss": 3.4324988658481494, + "tokens_seen": 1985347584 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040523232323232323, + "loss": 1.5827, + "theoretical_loss": 3.43248940516051, + "tokens_seen": 1985413120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052121212121212, + "loss": 1.5105, + "theoretical_loss": 3.432479944872588, + "tokens_seen": 1985478656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051919191919192, + "loss": 1.4755, + "theoretical_loss": 3.4324704849843526, + "tokens_seen": 1985544192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051717171717172, + "loss": 1.5322, + "theoretical_loss": 3.4324610254957744, + "tokens_seen": 1985609728 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040515151515151517, + "loss": 1.359, + "theoretical_loss": 3.4324515664068223, + "tokens_seen": 1985675264 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051313131313131, + "loss": 1.3518, + "theoretical_loss": 3.432442107717467, + "tokens_seen": 1985740800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051111111111111, + "loss": 1.5694, + "theoretical_loss": 3.4324326494276782, + "tokens_seen": 1985806336 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040509090909090915, + "loss": 1.2736, + "theoretical_loss": 3.432423191537426, + "tokens_seen": 1985871872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050707070707071, + "loss": 1.4792, + "theoretical_loss": 3.4324137340466807, + "tokens_seen": 1985937408 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.2547653913497925, + "objective/train/avg_token_score": 0.016683774068951607, + "objective/train/docs_used": 1123893, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.856845498085022, + "objective/train/original_loss": 5.153318881988525, + "objective/train/theoretical_loss": 3.432404276955411, + "objective/train/tokens_used": 344862176, + "objective/train/unlikelihood_loss": 0.05154005065560341, + "theoretical_loss": 3.432404276955411, + "tokens_seen": 1986002944 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040505050505050506, + "loss": 1.2664, + "theoretical_loss": 3.432404276955411, + "tokens_seen": 1986002944 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405030303030303, + "loss": 1.6352, + "theoretical_loss": 3.432394820263588, + "tokens_seen": 1986068480 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040501010101010097, + "loss": 1.4826, + "theoretical_loss": 3.432385363971181, + "tokens_seen": 1986134016 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040498989898989904, + "loss": 1.3789, + "theoretical_loss": 3.4323759080781606, + "tokens_seen": 1986199552 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404969696969697, + "loss": 1.4896, + "theoretical_loss": 3.432366452584496, + "tokens_seen": 1986265088 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040494949494949495, + "loss": 1.5823, + "theoretical_loss": 3.4323569974901575, + "tokens_seen": 1986330624 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049292929292929, + "loss": 1.5142, + "theoretical_loss": 3.4323475427951156, + "tokens_seen": 1986396160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049090909090909, + "loss": 1.4073, + "theoretical_loss": 3.4323380884993395, + "tokens_seen": 1986461696 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040488888888888893, + "loss": 1.4661, + "theoretical_loss": 3.4323286346027997, + "tokens_seen": 1986527232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048686868686869, + "loss": 1.5341, + "theoretical_loss": 3.4323191811054654, + "tokens_seen": 1986592768 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040484848484848484, + "loss": 1.314, + "theoretical_loss": 3.432309728007307, + "tokens_seen": 1986658304 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048282828282828, + "loss": 1.2981, + "theoretical_loss": 3.432300275308295, + "tokens_seen": 1986723840 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048080808080808, + "loss": 1.372, + "theoretical_loss": 3.4322908230083984, + "tokens_seen": 1986789376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047878787878788, + "loss": 1.4414, + "theoretical_loss": 3.432281371107588, + "tokens_seen": 1986854912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047676767676768, + "loss": 1.6455, + "theoretical_loss": 3.4322719196058338, + "tokens_seen": 1986920448 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040474747474747474, + "loss": 1.3235, + "theoretical_loss": 3.4322624685031053, + "tokens_seen": 1986985984 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040472727272727275, + "loss": 1.3656, + "theoretical_loss": 3.4322530177993724, + "tokens_seen": 1987051520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047070707070707, + "loss": 1.3143, + "theoretical_loss": 3.4322435674946057, + "tokens_seen": 1987117056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046868686868687, + "loss": 1.6096, + "theoretical_loss": 3.4322341175887745, + "tokens_seen": 1987182592 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040466666666666667, + "loss": 1.2566, + "theoretical_loss": 3.4322246680818496, + "tokens_seen": 1987248128 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040464646464646463, + "loss": 1.2232, + "theoretical_loss": 3.4322152189738, + "tokens_seen": 1987313664 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040462626262626264, + "loss": 1.5018, + "theoretical_loss": 3.4322057702645967, + "tokens_seen": 1987379200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046060606060606, + "loss": 1.5121, + "theoretical_loss": 3.4321963219542098, + "tokens_seen": 1987444736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045858585858586, + "loss": 1.3607, + "theoretical_loss": 3.4321868740426082, + "tokens_seen": 1987510272 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040456565656565657, + "loss": 1.818, + "theoretical_loss": 3.432177426529763, + "tokens_seen": 1987575808 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.5406891703605652, + "objective/train/avg_token_score": 0.005704171024262905, + "objective/train/docs_used": 1124511, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8316115140914917, + "objective/train/original_loss": 3.5460124015808105, + "objective/train/theoretical_loss": 3.4321679794156434, + "objective/train/tokens_used": 346500576, + "objective/train/unlikelihood_loss": 0.07432541251182556, + "theoretical_loss": 3.4321679794156434, + "tokens_seen": 1987641344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045454545454546, + "loss": 1.5832, + "theoretical_loss": 3.4321679794156434, + "tokens_seen": 1987641344 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040452525252525253, + "loss": 1.3207, + "theoretical_loss": 3.4321585327002198, + "tokens_seen": 1987706880 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045050505050505, + "loss": 1.3359, + "theoretical_loss": 3.432149086383462, + "tokens_seen": 1987772416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044848484848485, + "loss": 1.4143, + "theoretical_loss": 3.4321396404653406, + "tokens_seen": 1987837952 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040446464646464646, + "loss": 1.3399, + "theoretical_loss": 3.4321301949458256, + "tokens_seen": 1987903488 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040444444444444447, + "loss": 1.4967, + "theoretical_loss": 3.432120749824886, + "tokens_seen": 1987969024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044242424242424, + "loss": 1.6424, + "theoretical_loss": 3.4321113051024934, + "tokens_seen": 1988034560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044040404040404, + "loss": 1.656, + "theoretical_loss": 3.4321018607786167, + "tokens_seen": 1988100096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004043838383838384, + "loss": 1.3635, + "theoretical_loss": 3.4320924168532256, + "tokens_seen": 1988165632 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040436363636363635, + "loss": 1.4125, + "theoretical_loss": 3.432082973326292, + "tokens_seen": 1988231168 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040434343434343436, + "loss": 1.5523, + "theoretical_loss": 3.432073530197784, + "tokens_seen": 1988296704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004043232323232323, + "loss": 1.4857, + "theoretical_loss": 3.4320640874676727, + "tokens_seen": 1988362240 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040430303030303033, + "loss": 1.3104, + "theoretical_loss": 3.4320546451359277, + "tokens_seen": 1988427776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042828282828283, + "loss": 1.4525, + "theoretical_loss": 3.4320452032025193, + "tokens_seen": 1988493312 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042626262626263, + "loss": 1.4042, + "theoretical_loss": 3.4320357616674175, + "tokens_seen": 1988558848 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040424242424242425, + "loss": 1.387, + "theoretical_loss": 3.4320263205305928, + "tokens_seen": 1988624384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042222222222222, + "loss": 1.398, + "theoretical_loss": 3.4320168797920143, + "tokens_seen": 1988689920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042020202020202, + "loss": 1.3234, + "theoretical_loss": 3.432007439451653, + "tokens_seen": 1988755456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041818181818182, + "loss": 1.6412, + "theoretical_loss": 3.4319979995094787, + "tokens_seen": 1988820992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041616161616162, + "loss": 1.5261, + "theoretical_loss": 3.431988559965461, + "tokens_seen": 1988886528 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040414141414141415, + "loss": 1.5544, + "theoretical_loss": 3.431979120819571, + "tokens_seen": 1988952064 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041212121212121, + "loss": 1.4711, + "theoretical_loss": 3.4319696820717778, + "tokens_seen": 1989017600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041010101010101, + "loss": 1.2866, + "theoretical_loss": 3.431960243722052, + "tokens_seen": 1989083136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040808080808081, + "loss": 1.435, + "theoretical_loss": 3.4319508057703634, + "tokens_seen": 1989148672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040606060606061, + "loss": 1.4636, + "theoretical_loss": 3.4319413682166826, + "tokens_seen": 1989214208 + }, + { + "debugging/Self-BLEU-5": 0.4683363022964717, + "debugging/distinct-1-grams": 0.8034092501484725, + "debugging/distinct-2-grams": 0.9641002767459668, + "debugging/entropy-1-grams": 6.198537273223117, + "debugging/entropy-2-grams": 7.019141508423205, + "debugging/length": 468.3529411764706, + "debugging/num_segments": 17, + "epoch": 0.2, + "objective/train/aligned_token_count": 0.21560361981391907, + "objective/train/avg_token_score": 0.04828590154647827, + "objective/train/docs_used": 1125297, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8591703772544861, + "objective/train/original_loss": 5.175046443939209, + "objective/train/theoretical_loss": 3.431931931060979, + "objective/train/tokens_used": 348138976, + "objective/train/unlikelihood_loss": 0.05250702425837517, + "theoretical_loss": 3.431931931060979, + "tokens_seen": 1989279744 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040404040404040404, + "loss": 1.3473, + "theoretical_loss": 3.431931931060979, + "tokens_seen": 1989279744 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404020202020202, + "loss": 1.4626, + "theoretical_loss": 3.4319224943032234, + "tokens_seen": 1989345280 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404, + "loss": 1.278, + "theoretical_loss": 3.431913057943386, + "tokens_seen": 1989410816 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403979797979798, + "loss": 1.5107, + "theoretical_loss": 3.4319036219814363, + "tokens_seen": 1989476352 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403959595959596, + "loss": 1.5861, + "theoretical_loss": 3.4318941864173445, + "tokens_seen": 1989541888 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040393939393939393, + "loss": 1.2866, + "theoretical_loss": 3.4318847512510806, + "tokens_seen": 1989607424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039191919191919, + "loss": 1.5729, + "theoretical_loss": 3.431875316482615, + "tokens_seen": 1989672960 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040389898989898995, + "loss": 1.3685, + "theoretical_loss": 3.4318658821119183, + "tokens_seen": 1989738496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004038787878787879, + "loss": 1.5106, + "theoretical_loss": 3.43185644813896, + "tokens_seen": 1989804032 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040385858585858587, + "loss": 1.2137, + "theoretical_loss": 3.43184701456371, + "tokens_seen": 1989869568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004038383838383838, + "loss": 1.449, + "theoretical_loss": 3.431837581386139, + "tokens_seen": 1989935104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004038181818181818, + "loss": 1.2781, + "theoretical_loss": 3.4318281486062174, + "tokens_seen": 1990000640 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040379797979797985, + "loss": 1.7174, + "theoretical_loss": 3.4318187162239147, + "tokens_seen": 1990066176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037777777777778, + "loss": 1.3277, + "theoretical_loss": 3.4318092842392005, + "tokens_seen": 1990131712 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040375757575757576, + "loss": 1.4206, + "theoretical_loss": 3.4317998526520466, + "tokens_seen": 1990197248 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037373737373737, + "loss": 1.425, + "theoretical_loss": 3.4317904214624217, + "tokens_seen": 1990262784 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040371717171717173, + "loss": 1.4692, + "theoretical_loss": 3.4317809906702967, + "tokens_seen": 1990328320 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040369696969696974, + "loss": 1.345, + "theoretical_loss": 3.4317715602756413, + "tokens_seen": 1990393856 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036767676767677, + "loss": 1.3247, + "theoretical_loss": 3.4317621302784262, + "tokens_seen": 1990459392 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040365656565656565, + "loss": 1.5162, + "theoretical_loss": 3.4317527006786217, + "tokens_seen": 1990524928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036363636363636, + "loss": 1.4397, + "theoretical_loss": 3.431743271476197, + "tokens_seen": 1990590464 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036161616161616, + "loss": 1.5959, + "theoretical_loss": 3.4317338426711226, + "tokens_seen": 1990656000 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040359595959595963, + "loss": 1.3966, + "theoretical_loss": 3.4317244142633694, + "tokens_seen": 1990721536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035757575757576, + "loss": 1.3737, + "theoretical_loss": 3.4317149862529073, + "tokens_seen": 1990787072 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040355555555555555, + "loss": 1.4397, + "theoretical_loss": 3.4317055586397056, + "tokens_seen": 1990852608 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.2753543555736542, + "objective/train/avg_token_score": 0.040420789271593094, + "objective/train/docs_used": 1126404, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8407698273658752, + "objective/train/original_loss": 5.04019832611084, + "objective/train/theoretical_loss": 3.4316961314237355, + "objective/train/tokens_used": 349777376, + "objective/train/unlikelihood_loss": 0.060525793582201004, + "theoretical_loss": 3.4316961314237355, + "tokens_seen": 1990918144 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040353535353535356, + "loss": 1.468, + "theoretical_loss": 3.4316961314237355, + "tokens_seen": 1990918144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035151515151515, + "loss": 1.349, + "theoretical_loss": 3.431686704604967, + "tokens_seen": 1990983680 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034949494949495, + "loss": 1.2932, + "theoretical_loss": 3.4316772781833693, + "tokens_seen": 1991049216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034747474747475, + "loss": 1.5111, + "theoretical_loss": 3.431667852158914, + "tokens_seen": 1991114752 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040345454545454544, + "loss": 1.5132, + "theoretical_loss": 3.4316584265315706, + "tokens_seen": 1991180288 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040343434343434345, + "loss": 1.6366, + "theoretical_loss": 3.43164900130131, + "tokens_seen": 1991245824 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034141414141414, + "loss": 1.3234, + "theoretical_loss": 3.4316395764681014, + "tokens_seen": 1991311360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033939393939394, + "loss": 1.3436, + "theoretical_loss": 3.4316301520319152, + "tokens_seen": 1991376896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033737373737374, + "loss": 1.3677, + "theoretical_loss": 3.4316207279927218, + "tokens_seen": 1991442432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033535353535354, + "loss": 1.3794, + "theoretical_loss": 3.4316113043504917, + "tokens_seen": 1991507968 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040333333333333334, + "loss": 1.1624, + "theoretical_loss": 3.4316018811051947, + "tokens_seen": 1991573504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033131313131313, + "loss": 1.4437, + "theoretical_loss": 3.4315924582568016, + "tokens_seen": 1991639040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032929292929293, + "loss": 1.0514, + "theoretical_loss": 3.4315830358052817, + "tokens_seen": 1991704576 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040327272727272727, + "loss": 1.433, + "theoretical_loss": 3.431573613750606, + "tokens_seen": 1991770112 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032525252525253, + "loss": 1.3241, + "theoretical_loss": 3.431564192092744, + "tokens_seen": 1991835648 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040323232323232324, + "loss": 1.4644, + "theoretical_loss": 3.431554770831667, + "tokens_seen": 1991901184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032121212121212, + "loss": 1.4245, + "theoretical_loss": 3.4315453499673443, + "tokens_seen": 1991966720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031919191919192, + "loss": 1.5533, + "theoretical_loss": 3.4315359294997467, + "tokens_seen": 1992032256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031717171717172, + "loss": 1.4905, + "theoretical_loss": 3.431526509428844, + "tokens_seen": 1992097792 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040315151515151517, + "loss": 1.3922, + "theoretical_loss": 3.431517089754607, + "tokens_seen": 1992163328 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040313131313131313, + "loss": 1.4556, + "theoretical_loss": 3.4315076704770053, + "tokens_seen": 1992228864 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031111111111111, + "loss": 1.4792, + "theoretical_loss": 3.4314982515960093, + "tokens_seen": 1992294400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030909090909091, + "loss": 1.7444, + "theoretical_loss": 3.4314888331115894, + "tokens_seen": 1992359936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030707070707071, + "loss": 1.3701, + "theoretical_loss": 3.4314794150237162, + "tokens_seen": 1992425472 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040305050505050506, + "loss": 1.2929, + "theoretical_loss": 3.4314699973323597, + "tokens_seen": 1992491008 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.40243157744407654, + "objective/train/avg_token_score": 0.0219747181981802, + "objective/train/docs_used": 1126980, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4165253639221191, + "objective/train/original_loss": 4.742091655731201, + "objective/train/theoretical_loss": 3.4314605800374895, + "objective/train/tokens_used": 351415776, + "objective/train/unlikelihood_loss": 0.05318494141101837, + "theoretical_loss": 3.4314605800374895, + "tokens_seen": 1992556544 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403030303030303, + "loss": 1.4698, + "theoretical_loss": 3.4314605800374895, + "tokens_seen": 1992556544 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403010101010101, + "loss": 1.3088, + "theoretical_loss": 3.4314511631390774, + "tokens_seen": 1992622080 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040298989898989904, + "loss": 1.3889, + "theoretical_loss": 3.431441746637092, + "tokens_seen": 1992687616 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402969696969697, + "loss": 1.4465, + "theoretical_loss": 3.4314323305315044, + "tokens_seen": 1992753152 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040294949494949496, + "loss": 1.3891, + "theoretical_loss": 3.431422914822285, + "tokens_seen": 1992818688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029292929292929, + "loss": 1.5381, + "theoretical_loss": 3.431413499509404, + "tokens_seen": 1992884224 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040290909090909087, + "loss": 1.5889, + "theoretical_loss": 3.431404084592831, + "tokens_seen": 1992949760 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040288888888888894, + "loss": 1.3384, + "theoretical_loss": 3.4313946700725375, + "tokens_seen": 1993015296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028686868686869, + "loss": 1.1691, + "theoretical_loss": 3.4313852559484928, + "tokens_seen": 1993080832 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040284848484848485, + "loss": 1.2463, + "theoretical_loss": 3.4313758422206675, + "tokens_seen": 1993146368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028282828282828, + "loss": 1.4076, + "theoretical_loss": 3.4313664288890324, + "tokens_seen": 1993211904 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040280808080808076, + "loss": 1.5663, + "theoretical_loss": 3.431357015953557, + "tokens_seen": 1993277440 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040278787878787883, + "loss": 1.505, + "theoretical_loss": 3.4313476034142116, + "tokens_seen": 1993342976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027676767676768, + "loss": 1.3878, + "theoretical_loss": 3.431338191270967, + "tokens_seen": 1993408512 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040274747474747474, + "loss": 1.7233, + "theoretical_loss": 3.4313287795237932, + "tokens_seen": 1993474048 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027272727272727, + "loss": 1.4578, + "theoretical_loss": 3.4313193681726606, + "tokens_seen": 1993539584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027070707070707, + "loss": 1.3499, + "theoretical_loss": 3.4313099572175405, + "tokens_seen": 1993605120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026868686868687, + "loss": 1.5321, + "theoretical_loss": 3.4313005466584015, + "tokens_seen": 1993670656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026666666666667, + "loss": 1.6351, + "theoretical_loss": 3.431291136495215, + "tokens_seen": 1993736192 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040264646464646464, + "loss": 1.1717, + "theoretical_loss": 3.4312817267279505, + "tokens_seen": 1993801728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026262626262626, + "loss": 1.3782, + "theoretical_loss": 3.431272317356579, + "tokens_seen": 1993867264 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026060606060606, + "loss": 1.5529, + "theoretical_loss": 3.431262908381071, + "tokens_seen": 1993932800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025858585858586, + "loss": 1.255, + "theoretical_loss": 3.4312534998013966, + "tokens_seen": 1993998336 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040256565656565657, + "loss": 1.4759, + "theoretical_loss": 3.431244091617526, + "tokens_seen": 1994063872 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040254545454545453, + "loss": 1.3079, + "theoretical_loss": 3.4312346838294294, + "tokens_seen": 1994129408 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.42699170112609863, + "objective/train/avg_token_score": 0.011511247605085373, + "objective/train/docs_used": 1128284, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.334111213684082, + "objective/train/original_loss": 3.9049336910247803, + "objective/train/theoretical_loss": 3.431225276437078, + "objective/train/tokens_used": 353054176, + "objective/train/unlikelihood_loss": 0.11700768768787384, + "theoretical_loss": 3.431225276437078, + "tokens_seen": 1994194944 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040252525252525254, + "loss": 1.3111, + "theoretical_loss": 3.431225276437078, + "tokens_seen": 1994194944 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040250505050505055, + "loss": 1.3331, + "theoretical_loss": 3.431215869440441, + "tokens_seen": 1994260480 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024848484848485, + "loss": 1.3455, + "theoretical_loss": 3.431206462839489, + "tokens_seen": 1994326016 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040246464646464646, + "loss": 1.5316, + "theoretical_loss": 3.4311970566341934, + "tokens_seen": 1994391552 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024444444444444, + "loss": 1.3737, + "theoretical_loss": 3.4311876508245227, + "tokens_seen": 1994457088 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040242424242424243, + "loss": 1.1763, + "theoretical_loss": 3.4311782454104494, + "tokens_seen": 1994522624 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040240404040404044, + "loss": 1.65, + "theoretical_loss": 3.4311688403919423, + "tokens_seen": 1994588160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023838383838384, + "loss": 1.3469, + "theoretical_loss": 3.431159435768972, + "tokens_seen": 1994653696 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040236363636363636, + "loss": 1.2375, + "theoretical_loss": 3.4311500315415095, + "tokens_seen": 1994719232 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040234343434343437, + "loss": 1.7511, + "theoretical_loss": 3.431140627709525, + "tokens_seen": 1994784768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023232323232323, + "loss": 1.3529, + "theoretical_loss": 3.4311312242729883, + "tokens_seen": 1994850304 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040230303030303034, + "loss": 1.8496, + "theoretical_loss": 3.43112182123187, + "tokens_seen": 1994915840 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022828282828283, + "loss": 1.4838, + "theoretical_loss": 3.431112418586141, + "tokens_seen": 1994981376 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040226262626262625, + "loss": 1.6903, + "theoretical_loss": 3.431103016335771, + "tokens_seen": 1995046912 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040224242424242426, + "loss": 1.2681, + "theoretical_loss": 3.4310936144807314, + "tokens_seen": 1995112448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022222222222222, + "loss": 1.1243, + "theoretical_loss": 3.4310842130209913, + "tokens_seen": 1995177984 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040220202020202023, + "loss": 1.6698, + "theoretical_loss": 3.431074811956522, + "tokens_seen": 1995243520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021818181818182, + "loss": 1.5313, + "theoretical_loss": 3.4310654112872934, + "tokens_seen": 1995309056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021616161616162, + "loss": 1.3351, + "theoretical_loss": 3.4310560110132764, + "tokens_seen": 1995374592 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040214141414141415, + "loss": 1.5461, + "theoretical_loss": 3.431046611134441, + "tokens_seen": 1995440128 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021212121212121, + "loss": 1.3952, + "theoretical_loss": 3.4310372116507573, + "tokens_seen": 1995505664 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021010101010101, + "loss": 1.34, + "theoretical_loss": 3.4310278125621965, + "tokens_seen": 1995571200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020808080808081, + "loss": 1.3774, + "theoretical_loss": 3.431018413868729, + "tokens_seen": 1995636736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020606060606061, + "loss": 1.5431, + "theoretical_loss": 3.4310090155703246, + "tokens_seen": 1995702272 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040204040404040405, + "loss": 1.535, + "theoretical_loss": 3.430999617666954, + "tokens_seen": 1995767808 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.4018206298351288, + "objective/train/avg_token_score": 0.02627016417682171, + "objective/train/docs_used": 1128921, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3620922565460205, + "objective/train/original_loss": 4.719934463500977, + "objective/train/theoretical_loss": 3.430990220158588, + "objective/train/tokens_used": 354692576, + "objective/train/unlikelihood_loss": 0.050740428268909454, + "theoretical_loss": 3.430990220158588, + "tokens_seen": 1995833344 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402020202020202, + "loss": 1.5693, + "theoretical_loss": 3.430990220158588, + "tokens_seen": 1995833344 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402, + "loss": 1.3973, + "theoretical_loss": 3.4309808230451955, + "tokens_seen": 1995898880 + }, + { + "epoch": 0.2, + "learning_rate": 0.000401979797979798, + "loss": 1.3402, + "theoretical_loss": 3.430971426326749, + "tokens_seen": 1995964416 + }, + { + "epoch": 0.2, + "learning_rate": 0.000401959595959596, + "loss": 1.4646, + "theoretical_loss": 3.4309620300032178, + "tokens_seen": 1996029952 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040193939393939394, + "loss": 1.5222, + "theoretical_loss": 3.430952634074573, + "tokens_seen": 1996095488 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019191919191919, + "loss": 1.4229, + "theoretical_loss": 3.4309432385407836, + "tokens_seen": 1996161024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018989898989899, + "loss": 1.3307, + "theoretical_loss": 3.4309338434018217, + "tokens_seen": 1996226560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018787878787879, + "loss": 1.6634, + "theoretical_loss": 3.430924448657657, + "tokens_seen": 1996292096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018585858585859, + "loss": 1.394, + "theoretical_loss": 3.43091505430826, + "tokens_seen": 1996357632 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040183838383838383, + "loss": 1.5225, + "theoretical_loss": 3.4309056603536012, + "tokens_seen": 1996423168 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018181818181818, + "loss": 1.4662, + "theoretical_loss": 3.430896266793651, + "tokens_seen": 1996488704 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040179797979797985, + "loss": 1.4994, + "theoretical_loss": 3.43088687362838, + "tokens_seen": 1996554240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017777777777778, + "loss": 1.3143, + "theoretical_loss": 3.4308774808577587, + "tokens_seen": 1996619776 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040175757575757577, + "loss": 1.3971, + "theoretical_loss": 3.4308680884817573, + "tokens_seen": 1996685312 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017373737373737, + "loss": 1.4626, + "theoretical_loss": 3.430858696500346, + "tokens_seen": 1996750848 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017171717171717, + "loss": 1.5061, + "theoretical_loss": 3.430849304913496, + "tokens_seen": 1996816384 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040169696969696975, + "loss": 1.4394, + "theoretical_loss": 3.4308399137211776, + "tokens_seen": 1996881920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016767676767677, + "loss": 1.6052, + "theoretical_loss": 3.430830522923361, + "tokens_seen": 1996947456 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040165656565656566, + "loss": 1.4002, + "theoretical_loss": 3.4308211325200166, + "tokens_seen": 1997012992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016363636363636, + "loss": 1.5824, + "theoretical_loss": 3.430811742511115, + "tokens_seen": 1997078528 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040161616161616163, + "loss": 1.5992, + "theoretical_loss": 3.4308023528966274, + "tokens_seen": 1997144064 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040159595959595964, + "loss": 1.5798, + "theoretical_loss": 3.4307929636765233, + "tokens_seen": 1997209600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015757575757576, + "loss": 1.6261, + "theoretical_loss": 3.4307835748507736, + "tokens_seen": 1997275136 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040155555555555555, + "loss": 1.4311, + "theoretical_loss": 3.430774186419349, + "tokens_seen": 1997340672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015353535353535, + "loss": 1.3639, + "theoretical_loss": 3.4307647983822194, + "tokens_seen": 1997406208 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.4560728371143341, + "objective/train/avg_token_score": 0.014928154647350311, + "objective/train/docs_used": 1129917, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6200169324874878, + "objective/train/original_loss": 4.3188934326171875, + "objective/train/theoretical_loss": 3.430755410739356, + "objective/train/tokens_used": 356330976, + "objective/train/unlikelihood_loss": 0.06730559468269348, + "theoretical_loss": 3.430755410739356, + "tokens_seen": 1997471744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015151515151515, + "loss": 1.5374, + "theoretical_loss": 3.430755410739356, + "tokens_seen": 1997471744 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040149494949494953, + "loss": 1.5942, + "theoretical_loss": 3.4307460234907285, + "tokens_seen": 1997537280 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014747474747475, + "loss": 1.2555, + "theoretical_loss": 3.430736636636308, + "tokens_seen": 1997602816 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040145454545454545, + "loss": 1.2842, + "theoretical_loss": 3.430727250176065, + "tokens_seen": 1997668352 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040143434343434346, + "loss": 1.2268, + "theoretical_loss": 3.4307178641099703, + "tokens_seen": 1997733888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014141414141414, + "loss": 1.4628, + "theoretical_loss": 3.4307084784379933, + "tokens_seen": 1997799424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013939393939394, + "loss": 1.5295, + "theoretical_loss": 3.430699093160106, + "tokens_seen": 1997864960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013737373737374, + "loss": 1.3422, + "theoretical_loss": 3.4306897082762777, + "tokens_seen": 1997930496 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040135353535353534, + "loss": 1.3854, + "theoretical_loss": 3.4306803237864796, + "tokens_seen": 1997996032 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040133333333333335, + "loss": 1.3874, + "theoretical_loss": 3.4306709396906823, + "tokens_seen": 1998061568 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013131313131313, + "loss": 1.5439, + "theoretical_loss": 3.4306615559888556, + "tokens_seen": 1998127104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012929292929293, + "loss": 1.3111, + "theoretical_loss": 3.4306521726809707, + "tokens_seen": 1998192640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012727272727273, + "loss": 1.514, + "theoretical_loss": 3.430642789766998, + "tokens_seen": 1998258176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012525252525253, + "loss": 1.4821, + "theoretical_loss": 3.4306334072469085, + "tokens_seen": 1998323712 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040123232323232324, + "loss": 1.4354, + "theoretical_loss": 3.4306240251206717, + "tokens_seen": 1998389248 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012121212121212, + "loss": 1.5244, + "theoretical_loss": 3.4306146433882585, + "tokens_seen": 1998454784 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011919191919192, + "loss": 1.6676, + "theoretical_loss": 3.43060526204964, + "tokens_seen": 1998520320 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040117171717171717, + "loss": 1.4527, + "theoretical_loss": 3.4305958811047863, + "tokens_seen": 1998585856 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011515151515152, + "loss": 1.7453, + "theoretical_loss": 3.4305865005536686, + "tokens_seen": 1998651392 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040113131313131313, + "loss": 1.3844, + "theoretical_loss": 3.430577120396256, + "tokens_seen": 1998716928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011111111111111, + "loss": 1.4724, + "theoretical_loss": 3.4305677406325206, + "tokens_seen": 1998782464 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004010909090909091, + "loss": 1.5013, + "theoretical_loss": 3.430558361262433, + "tokens_seen": 1998848000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040107070707070706, + "loss": 1.6508, + "theoretical_loss": 3.430548982285962, + "tokens_seen": 1998913536 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040105050505050507, + "loss": 1.5671, + "theoretical_loss": 3.43053960370308, + "tokens_seen": 1998979072 + }, + { + "epoch": 0.21, + "learning_rate": 0.000401030303030303, + "loss": 1.3459, + "theoretical_loss": 3.4305302255137566, + "tokens_seen": 1999044608 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.5745356678962708, + "objective/train/avg_token_score": 0.00940377451479435, + "objective/train/docs_used": 1130465, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.279651403427124, + "objective/train/original_loss": 3.9225895404815674, + "objective/train/theoretical_loss": 3.430520847717963, + "objective/train/tokens_used": 357969376, + "objective/train/unlikelihood_loss": 0.0918278694152832, + "theoretical_loss": 3.430520847717963, + "tokens_seen": 1999110144 + }, + { + "epoch": 0.21, + "learning_rate": 0.000401010101010101, + "loss": 1.5423, + "theoretical_loss": 3.430520847717963, + "tokens_seen": 1999110144 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400989898989899, + "loss": 1.7275, + "theoretical_loss": 3.430511470315669, + "tokens_seen": 1999175680 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400969696969697, + "loss": 1.2737, + "theoretical_loss": 3.4305020933068464, + "tokens_seen": 1999241216 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040094949494949496, + "loss": 1.4778, + "theoretical_loss": 3.4304927166914645, + "tokens_seen": 1999306752 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009292929292929, + "loss": 1.6099, + "theoretical_loss": 3.430483340469495, + "tokens_seen": 1999372288 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009090909090909, + "loss": 1.5138, + "theoretical_loss": 3.4304739646409077, + "tokens_seen": 1999437824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008888888888889, + "loss": 1.3048, + "theoretical_loss": 3.4304645892056733, + "tokens_seen": 1999503360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008686868686869, + "loss": 1.2571, + "theoretical_loss": 3.4304552141637625, + "tokens_seen": 1999568896 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040084848484848486, + "loss": 1.2765, + "theoretical_loss": 3.4304458395151465, + "tokens_seen": 1999634432 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008282828282828, + "loss": 1.223, + "theoretical_loss": 3.430436465259795, + "tokens_seen": 1999699968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008080808080808, + "loss": 1.5195, + "theoretical_loss": 3.4304270913976795, + "tokens_seen": 1999765504 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040078787878787883, + "loss": 1.2751, + "theoretical_loss": 3.430417717928769, + "tokens_seen": 1999831040 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007676767676768, + "loss": 1.3775, + "theoretical_loss": 3.4304083448530367, + "tokens_seen": 1999896576 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040074747474747475, + "loss": 1.3148, + "theoretical_loss": 3.4303989721704506, + "tokens_seen": 1999962112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007272727272727, + "loss": 1.3953, + "theoretical_loss": 3.430389599880983, + "tokens_seen": 2000027648 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007070707070707, + "loss": 1.2631, + "theoretical_loss": 3.430380227984604, + "tokens_seen": 2000093184 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040068686868686873, + "loss": 1.2196, + "theoretical_loss": 3.4303708564812845, + "tokens_seen": 2000158720 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006666666666667, + "loss": 1.3836, + "theoretical_loss": 3.4303614853709945, + "tokens_seen": 2000224256 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040064646464646464, + "loss": 1.2242, + "theoretical_loss": 3.4303521146537053, + "tokens_seen": 2000289792 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006262626262626, + "loss": 1.28, + "theoretical_loss": 3.430342744329387, + "tokens_seen": 2000355328 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040060606060606066, + "loss": 1.3343, + "theoretical_loss": 3.430333374398011, + "tokens_seen": 2000420864 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005858585858586, + "loss": 1.4141, + "theoretical_loss": 3.430324004859547, + "tokens_seen": 2000486400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005656565656566, + "loss": 1.0115, + "theoretical_loss": 3.430314635713967, + "tokens_seen": 2000551936 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040054545454545453, + "loss": 1.3156, + "theoretical_loss": 3.4303052669612395, + "tokens_seen": 2000617472 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005252525252525, + "loss": 1.7144, + "theoretical_loss": 3.4302958986013374, + "tokens_seen": 2000683008 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.3841642141342163, + "objective/train/avg_token_score": 0.048149384558200836, + "objective/train/docs_used": 1131148, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.318234920501709, + "objective/train/original_loss": 5.058935165405273, + "objective/train/theoretical_loss": 3.43028653063423, + "objective/train/tokens_used": 359607776, + "objective/train/unlikelihood_loss": 0.04659086838364601, + "theoretical_loss": 3.43028653063423, + "tokens_seen": 2000748544 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040050505050505056, + "loss": 1.4759, + "theoretical_loss": 3.43028653063423, + "tokens_seen": 2000748544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004848484848485, + "loss": 1.1273, + "theoretical_loss": 3.4302771630598885, + "tokens_seen": 2000814080 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040046464646464647, + "loss": 1.4563, + "theoretical_loss": 3.4302677958782835, + "tokens_seen": 2000879616 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004444444444444, + "loss": 1.294, + "theoretical_loss": 3.4302584290893856, + "tokens_seen": 2000945152 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040042424242424244, + "loss": 1.3029, + "theoretical_loss": 3.430249062693165, + "tokens_seen": 2001010688 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040040404040404045, + "loss": 1.4544, + "theoretical_loss": 3.430239696689594, + "tokens_seen": 2001076224 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003838383838384, + "loss": 1.4593, + "theoretical_loss": 3.4302303310786417, + "tokens_seen": 2001141760 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040036363636363636, + "loss": 1.2595, + "theoretical_loss": 3.4302209658602787, + "tokens_seen": 2001207296 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003434343434343, + "loss": 1.395, + "theoretical_loss": 3.4302116010344763, + "tokens_seen": 2001272832 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040032323232323233, + "loss": 1.2054, + "theoretical_loss": 3.4302022366012057, + "tokens_seen": 2001338368 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040030303030303034, + "loss": 1.3016, + "theoretical_loss": 3.4301928725604367, + "tokens_seen": 2001403904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004002828282828283, + "loss": 1.7202, + "theoretical_loss": 3.4301835089121404, + "tokens_seen": 2001469440 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040026262626262625, + "loss": 1.542, + "theoretical_loss": 3.430174145656287, + "tokens_seen": 2001534976 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040024242424242427, + "loss": 1.2283, + "theoretical_loss": 3.4301647827928483, + "tokens_seen": 2001600512 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004002222222222222, + "loss": 1.3033, + "theoretical_loss": 3.430155420321794, + "tokens_seen": 2001666048 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040020202020202023, + "loss": 1.4857, + "theoretical_loss": 3.430146058243095, + "tokens_seen": 2001731584 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001818181818182, + "loss": 1.4668, + "theoretical_loss": 3.430136696556722, + "tokens_seen": 2001797120 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040016161616161615, + "loss": 1.0117, + "theoretical_loss": 3.4301273352626462, + "tokens_seen": 2001862656 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040014141414141416, + "loss": 1.3209, + "theoretical_loss": 3.430117974360838, + "tokens_seen": 2001928192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001212121212121, + "loss": 1.4727, + "theoretical_loss": 3.4301086138512678, + "tokens_seen": 2001993728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001010101010101, + "loss": 1.6293, + "theoretical_loss": 3.430099253733907, + "tokens_seen": 2002059264 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000808080808081, + "loss": 1.6588, + "theoretical_loss": 3.4300898940087254, + "tokens_seen": 2002124800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000606060606061, + "loss": 1.2346, + "theoretical_loss": 3.430080534675695, + "tokens_seen": 2002190336 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040004040404040405, + "loss": 1.2629, + "theoretical_loss": 3.4300711757347853, + "tokens_seen": 2002255872 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400020202020202, + "loss": 1.2379, + "theoretical_loss": 3.4300618171859676, + "tokens_seen": 2002321408 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.39876589179039, + "objective/train/avg_token_score": 0.012741667218506336, + "objective/train/docs_used": 1132271, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2703096866607666, + "objective/train/original_loss": 3.7862753868103027, + "objective/train/theoretical_loss": 3.4300524590292127, + "objective/train/tokens_used": 361246176, + "objective/train/unlikelihood_loss": 0.11513296514749527, + "theoretical_loss": 3.4300524590292127, + "tokens_seen": 2002386944 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004, + "loss": 1.4336, + "theoretical_loss": 3.4300524590292127, + "tokens_seen": 2002386944 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399979797979798, + "loss": 1.1594, + "theoretical_loss": 3.4300431012644914, + "tokens_seen": 2002452480 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399959595959596, + "loss": 1.5808, + "theoretical_loss": 3.430033743891774, + "tokens_seen": 2002518016 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039993939393939394, + "loss": 1.4235, + "theoretical_loss": 3.4300243869110316, + "tokens_seen": 2002583552 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999191919191919, + "loss": 1.4838, + "theoretical_loss": 3.430015030322235, + "tokens_seen": 2002649088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998989898989899, + "loss": 1.3642, + "theoretical_loss": 3.4300056741253546, + "tokens_seen": 2002714624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998787878787879, + "loss": 1.4876, + "theoretical_loss": 3.4299963183203617, + "tokens_seen": 2002780160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998585858585859, + "loss": 1.5863, + "theoretical_loss": 3.4299869629072264, + "tokens_seen": 2002845696 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039983838383838384, + "loss": 1.5689, + "theoretical_loss": 3.4299776078859203, + "tokens_seen": 2002911232 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998181818181818, + "loss": 1.1708, + "theoretical_loss": 3.4299682532564137, + "tokens_seen": 2002976768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003997979797979798, + "loss": 1.1919, + "theoretical_loss": 3.4299588990186773, + "tokens_seen": 2003042304 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003997777777777778, + "loss": 1.4953, + "theoretical_loss": 3.429949545172682, + "tokens_seen": 2003107840 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039975757575757577, + "loss": 1.3504, + "theoretical_loss": 3.429940191718398, + "tokens_seen": 2003173376 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039973737373737373, + "loss": 1.5487, + "theoretical_loss": 3.429930838655797, + "tokens_seen": 2003238912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003997171717171717, + "loss": 1.5136, + "theoretical_loss": 3.4299214859848495, + "tokens_seen": 2003304448 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039969696969696975, + "loss": 1.2122, + "theoretical_loss": 3.429912133705526, + "tokens_seen": 2003369984 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996767676767677, + "loss": 1.3716, + "theoretical_loss": 3.4299027818177974, + "tokens_seen": 2003435520 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039965656565656567, + "loss": 1.6886, + "theoretical_loss": 3.429893430321635, + "tokens_seen": 2003501056 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996363636363636, + "loss": 0.969, + "theoretical_loss": 3.4298840792170084, + "tokens_seen": 2003566592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996161616161616, + "loss": 1.4569, + "theoretical_loss": 3.42987472850389, + "tokens_seen": 2003632128 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039959595959595964, + "loss": 1.4078, + "theoretical_loss": 3.4298653781822495, + "tokens_seen": 2003697664 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995757575757576, + "loss": 1.3704, + "theoretical_loss": 3.429856028252058, + "tokens_seen": 2003763200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039955555555555556, + "loss": 1.3195, + "theoretical_loss": 3.429846678713286, + "tokens_seen": 2003828736 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995353535353535, + "loss": 1.4617, + "theoretical_loss": 3.4298373295659053, + "tokens_seen": 2003894272 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039951515151515147, + "loss": 1.4341, + "theoretical_loss": 3.4298279808098853, + "tokens_seen": 2003959808 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.291177898645401, + "objective/train/avg_token_score": 0.03287602216005325, + "objective/train/docs_used": 1132794, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9526001214981079, + "objective/train/original_loss": 5.209911823272705, + "objective/train/theoretical_loss": 3.4298186324451976, + "objective/train/tokens_used": 362884576, + "objective/train/unlikelihood_loss": 0.04816959425806999, + "theoretical_loss": 3.4298186324451976, + "tokens_seen": 2004025344 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039949494949494954, + "loss": 1.1675, + "theoretical_loss": 3.4298186324451976, + "tokens_seen": 2004025344 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994747474747475, + "loss": 1.4201, + "theoretical_loss": 3.4298092844718133, + "tokens_seen": 2004090880 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039945454545454545, + "loss": 1.2932, + "theoretical_loss": 3.4297999368897027, + "tokens_seen": 2004156416 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994343434343434, + "loss": 1.3336, + "theoretical_loss": 3.429790589698837, + "tokens_seen": 2004221952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994141414141414, + "loss": 1.3756, + "theoretical_loss": 3.4297812428991867, + "tokens_seen": 2004287488 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039939393939393943, + "loss": 1.5859, + "theoretical_loss": 3.429771896490723, + "tokens_seen": 2004353024 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993737373737374, + "loss": 1.6585, + "theoretical_loss": 3.4297625504734164, + "tokens_seen": 2004418560 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039935353535353534, + "loss": 1.1558, + "theoretical_loss": 3.4297532048472377, + "tokens_seen": 2004484096 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993333333333333, + "loss": 1.3409, + "theoretical_loss": 3.4297438596121577, + "tokens_seen": 2004549632 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993131313131313, + "loss": 1.4537, + "theoretical_loss": 3.429734514768148, + "tokens_seen": 2004615168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003992929292929293, + "loss": 1.3233, + "theoretical_loss": 3.4297251703151783, + "tokens_seen": 2004680704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003992727272727273, + "loss": 1.3178, + "theoretical_loss": 3.4297158262532204, + "tokens_seen": 2004746240 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039925252525252524, + "loss": 1.46, + "theoretical_loss": 3.4297064825822448, + "tokens_seen": 2004811776 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039923232323232325, + "loss": 1.1032, + "theoretical_loss": 3.4296971393022226, + "tokens_seen": 2004877312 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003992121212121212, + "loss": 1.4979, + "theoretical_loss": 3.4296877964131243, + "tokens_seen": 2004942848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003991919191919192, + "loss": 1.4713, + "theoretical_loss": 3.4296784539149208, + "tokens_seen": 2005008384 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039917171717171717, + "loss": 1.525, + "theoretical_loss": 3.4296691118075833, + "tokens_seen": 2005073920 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039915151515151513, + "loss": 1.1359, + "theoretical_loss": 3.4296597700910825, + "tokens_seen": 2005139456 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039913131313131314, + "loss": 1.364, + "theoretical_loss": 3.429650428765389, + "tokens_seen": 2005204992 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039911111111111115, + "loss": 1.0891, + "theoretical_loss": 3.4296410878304737, + "tokens_seen": 2005270528 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003990909090909091, + "loss": 1.1563, + "theoretical_loss": 3.429631747286308, + "tokens_seen": 2005336064 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039907070707070706, + "loss": 1.3294, + "theoretical_loss": 3.4296224071328623, + "tokens_seen": 2005401600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003990505050505051, + "loss": 1.498, + "theoretical_loss": 3.4296130673701075, + "tokens_seen": 2005467136 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039903030303030303, + "loss": 1.3699, + "theoretical_loss": 3.429603727998015, + "tokens_seen": 2005532672 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039901010101010104, + "loss": 1.3808, + "theoretical_loss": 3.429594389016555, + "tokens_seen": 2005598208 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.48179373145103455, + "objective/train/avg_token_score": 0.01661764457821846, + "objective/train/docs_used": 1134045, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4383306503295898, + "objective/train/original_loss": 4.333360195159912, + "objective/train/theoretical_loss": 3.429585050425699, + "objective/train/tokens_used": 364522976, + "objective/train/unlikelihood_loss": 0.05595013126730919, + "theoretical_loss": 3.429585050425699, + "tokens_seen": 2005663744 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398989898989899, + "loss": 1.3411, + "theoretical_loss": 3.429585050425699, + "tokens_seen": 2005663744 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039896969696969696, + "loss": 1.3543, + "theoretical_loss": 3.429575712225418, + "tokens_seen": 2005729280 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039894949494949497, + "loss": 1.3326, + "theoretical_loss": 3.429566374415682, + "tokens_seen": 2005794816 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003989292929292929, + "loss": 1.6406, + "theoretical_loss": 3.4295570369964623, + "tokens_seen": 2005860352 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039890909090909094, + "loss": 1.3855, + "theoretical_loss": 3.4295476999677303, + "tokens_seen": 2005925888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003988888888888889, + "loss": 1.3958, + "theoretical_loss": 3.4295383633294563, + "tokens_seen": 2005991424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003988686868686869, + "loss": 1.4905, + "theoretical_loss": 3.429529027081612, + "tokens_seen": 2006056960 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039884848484848486, + "loss": 1.5166, + "theoretical_loss": 3.4295196912241677, + "tokens_seen": 2006122496 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003988282828282828, + "loss": 1.6309, + "theoretical_loss": 3.429510355757094, + "tokens_seen": 2006188032 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039880808080808083, + "loss": 1.585, + "theoretical_loss": 3.4295010206803624, + "tokens_seen": 2006253568 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987878787878788, + "loss": 1.4573, + "theoretical_loss": 3.429491685993944, + "tokens_seen": 2006319104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987676767676768, + "loss": 1.6665, + "theoretical_loss": 3.4294823516978092, + "tokens_seen": 2006384640 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039874747474747475, + "loss": 1.2809, + "theoretical_loss": 3.4294730177919295, + "tokens_seen": 2006450176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987272727272727, + "loss": 1.6269, + "theoretical_loss": 3.429463684276275, + "tokens_seen": 2006515712 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987070707070707, + "loss": 1.7012, + "theoretical_loss": 3.429454351150817, + "tokens_seen": 2006581248 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039868686868686873, + "loss": 1.489, + "theoretical_loss": 3.429445018415527, + "tokens_seen": 2006646784 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986666666666667, + "loss": 1.5224, + "theoretical_loss": 3.4294356860703754, + "tokens_seen": 2006712320 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039864646464646465, + "loss": 1.4711, + "theoretical_loss": 3.429426354115334, + "tokens_seen": 2006777856 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986262626262626, + "loss": 1.2435, + "theoretical_loss": 3.4294170225503717, + "tokens_seen": 2006843392 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986060606060606, + "loss": 1.3795, + "theoretical_loss": 3.429407691375461, + "tokens_seen": 2006908928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985858585858586, + "loss": 1.4629, + "theoretical_loss": 3.4293983605905733, + "tokens_seen": 2006974464 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985656565656566, + "loss": 1.569, + "theoretical_loss": 3.429389030195679, + "tokens_seen": 2007040000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039854545454545454, + "loss": 1.3673, + "theoretical_loss": 3.4293797001907484, + "tokens_seen": 2007105536 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985252525252525, + "loss": 1.4482, + "theoretical_loss": 3.4293703705757537, + "tokens_seen": 2007171072 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039850505050505056, + "loss": 1.1585, + "theoretical_loss": 3.4293610413506643, + "tokens_seen": 2007236608 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.4596163332462311, + "objective/train/avg_token_score": 0.01717677153646946, + "objective/train/docs_used": 1134700, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4232748746871948, + "objective/train/original_loss": 4.048245429992676, + "objective/train/theoretical_loss": 3.4293517125154525, + "objective/train/tokens_used": 366161376, + "objective/train/unlikelihood_loss": 0.062166180461645126, + "theoretical_loss": 3.4293517125154525, + "tokens_seen": 2007302144 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984848484848485, + "loss": 1.3018, + "theoretical_loss": 3.4293517125154525, + "tokens_seen": 2007302144 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984646464646465, + "loss": 1.0955, + "theoretical_loss": 3.429342384070089, + "tokens_seen": 2007367680 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039844444444444443, + "loss": 1.2975, + "theoretical_loss": 3.4293330560145447, + "tokens_seen": 2007433216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984242424242424, + "loss": 1.5909, + "theoretical_loss": 3.4293237283487903, + "tokens_seen": 2007498752 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039840404040404045, + "loss": 1.2435, + "theoretical_loss": 3.4293144010727974, + "tokens_seen": 2007564288 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983838383838384, + "loss": 1.2224, + "theoretical_loss": 3.4293050741865363, + "tokens_seen": 2007629824 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039836363636363637, + "loss": 1.4889, + "theoretical_loss": 3.429295747689978, + "tokens_seen": 2007695360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983434343434343, + "loss": 1.6156, + "theoretical_loss": 3.4292864215830945, + "tokens_seen": 2007760896 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039832323232323234, + "loss": 1.2967, + "theoretical_loss": 3.4292770958658556, + "tokens_seen": 2007826432 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039830303030303035, + "loss": 1.4694, + "theoretical_loss": 3.4292677705382335, + "tokens_seen": 2007891968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982828282828283, + "loss": 1.5067, + "theoretical_loss": 3.4292584456001975, + "tokens_seen": 2007957504 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039826262626262626, + "loss": 1.389, + "theoretical_loss": 3.42924912105172, + "tokens_seen": 2008023040 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982424242424242, + "loss": 1.2721, + "theoretical_loss": 3.429239796892772, + "tokens_seen": 2008088576 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039822222222222223, + "loss": 1.4431, + "theoretical_loss": 3.429230473123324, + "tokens_seen": 2008154112 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039820202020202024, + "loss": 1.4681, + "theoretical_loss": 3.429221149743347, + "tokens_seen": 2008219648 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981818181818182, + "loss": 1.6894, + "theoretical_loss": 3.4292118267528124, + "tokens_seen": 2008285184 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039816161616161615, + "loss": 1.1353, + "theoretical_loss": 3.429202504151691, + "tokens_seen": 2008350720 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039814141414141416, + "loss": 1.6095, + "theoretical_loss": 3.4291931819399535, + "tokens_seen": 2008416256 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981212121212121, + "loss": 1.6248, + "theoretical_loss": 3.429183860117572, + "tokens_seen": 2008481792 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039810101010101013, + "loss": 1.436, + "theoretical_loss": 3.4291745386845163, + "tokens_seen": 2008547328 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980808080808081, + "loss": 1.2587, + "theoretical_loss": 3.429165217640758, + "tokens_seen": 2008612864 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039806060606060605, + "loss": 1.6218, + "theoretical_loss": 3.429155896986268, + "tokens_seen": 2008678400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039804040404040406, + "loss": 1.6518, + "theoretical_loss": 3.4291465767210174, + "tokens_seen": 2008743936 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398020202020202, + "loss": 1.5358, + "theoretical_loss": 3.429137256844977, + "tokens_seen": 2008809472 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398, + "loss": 1.4281, + "theoretical_loss": 3.429127937358119, + "tokens_seen": 2008875008 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.4087854325771332, + "objective/train/avg_token_score": 0.03126665577292442, + "objective/train/docs_used": 1135755, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2452770471572876, + "objective/train/original_loss": 4.195167541503906, + "objective/train/theoretical_loss": 3.429118618260413, + "objective/train/tokens_used": 367799776, + "objective/train/unlikelihood_loss": 0.037951596081256866, + "theoretical_loss": 3.429118618260413, + "tokens_seen": 2008940544 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397979797979798, + "loss": 1.351, + "theoretical_loss": 3.429118618260413, + "tokens_seen": 2008940544 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397959595959596, + "loss": 1.3136, + "theoretical_loss": 3.429109299551831, + "tokens_seen": 2009006080 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039793939393939395, + "loss": 1.1987, + "theoretical_loss": 3.4290999812323433, + "tokens_seen": 2009071616 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003979191919191919, + "loss": 1.4176, + "theoretical_loss": 3.4290906633019214, + "tokens_seen": 2009137152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978989898989899, + "loss": 1.6675, + "theoretical_loss": 3.4290813457605362, + "tokens_seen": 2009202688 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978787878787879, + "loss": 1.4206, + "theoretical_loss": 3.429072028608159, + "tokens_seen": 2009268224 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978585858585859, + "loss": 1.5396, + "theoretical_loss": 3.429062711844761, + "tokens_seen": 2009333760 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039783838383838384, + "loss": 1.4403, + "theoretical_loss": 3.429053395470313, + "tokens_seen": 2009399296 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978181818181818, + "loss": 1.2628, + "theoretical_loss": 3.429044079484786, + "tokens_seen": 2009464832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977979797979798, + "loss": 1.4167, + "theoretical_loss": 3.4290347638881515, + "tokens_seen": 2009530368 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039777777777777777, + "loss": 1.5803, + "theoretical_loss": 3.42902544868038, + "tokens_seen": 2009595904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977575757575758, + "loss": 1.5627, + "theoretical_loss": 3.4290161338614427, + "tokens_seen": 2009661440 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039773737373737373, + "loss": 1.4839, + "theoretical_loss": 3.4290068194313115, + "tokens_seen": 2009726976 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977171717171717, + "loss": 1.6212, + "theoretical_loss": 3.428997505389956, + "tokens_seen": 2009792512 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976969696969697, + "loss": 1.5033, + "theoretical_loss": 3.4289881917373486, + "tokens_seen": 2009858048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976767676767677, + "loss": 1.3601, + "theoretical_loss": 3.42897887847346, + "tokens_seen": 2009923584 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039765656565656567, + "loss": 1.4848, + "theoretical_loss": 3.428969565598261, + "tokens_seen": 2009989120 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039763636363636363, + "loss": 1.5622, + "theoretical_loss": 3.428960253111723, + "tokens_seen": 2010054656 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976161616161616, + "loss": 1.3449, + "theoretical_loss": 3.4289509410138175, + "tokens_seen": 2010120192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975959595959596, + "loss": 1.3104, + "theoretical_loss": 3.4289416293045147, + "tokens_seen": 2010185728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975757575757576, + "loss": 1.4158, + "theoretical_loss": 3.4289323179837865, + "tokens_seen": 2010251264 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039755555555555556, + "loss": 1.485, + "theoretical_loss": 3.4289230070516035, + "tokens_seen": 2010316800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975353535353535, + "loss": 1.5817, + "theoretical_loss": 3.428913696507937, + "tokens_seen": 2010382336 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975151515151515, + "loss": 1.3679, + "theoretical_loss": 3.428904386352758, + "tokens_seen": 2010447872 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039749494949494954, + "loss": 1.4348, + "theoretical_loss": 3.428895076586038, + "tokens_seen": 2010513408 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.3502565920352936, + "objective/train/avg_token_score": 0.01835888810455799, + "objective/train/docs_used": 1136767, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1449387073516846, + "objective/train/original_loss": 4.458366870880127, + "objective/train/theoretical_loss": 3.428885767207748, + "objective/train/tokens_used": 369438176, + "objective/train/unlikelihood_loss": 0.04602991044521332, + "theoretical_loss": 3.428885767207748, + "tokens_seen": 2010578944 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003974747474747475, + "loss": 1.2614, + "theoretical_loss": 3.428885767207748, + "tokens_seen": 2010578944 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039745454545454546, + "loss": 1.568, + "theoretical_loss": 3.428876458217859, + "tokens_seen": 2010644480 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003974343434343434, + "loss": 1.4913, + "theoretical_loss": 3.4288671496163423, + "tokens_seen": 2010710016 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039741414141414137, + "loss": 1.5301, + "theoretical_loss": 3.4288578414031687, + "tokens_seen": 2010775552 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039739393939393944, + "loss": 1.4495, + "theoretical_loss": 3.42884853357831, + "tokens_seen": 2010841088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973737373737374, + "loss": 1.4207, + "theoretical_loss": 3.4288392261417364, + "tokens_seen": 2010906624 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039735353535353535, + "loss": 1.2996, + "theoretical_loss": 3.4288299190934195, + "tokens_seen": 2010972160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973333333333333, + "loss": 1.4799, + "theoretical_loss": 3.4288206124333307, + "tokens_seen": 2011037696 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039731313131313137, + "loss": 1.6606, + "theoretical_loss": 3.4288113061614407, + "tokens_seen": 2011103232 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039729292929292933, + "loss": 1.4349, + "theoretical_loss": 3.4288020002777215, + "tokens_seen": 2011168768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972727272727273, + "loss": 1.3954, + "theoretical_loss": 3.4287926947821434, + "tokens_seen": 2011234304 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039725252525252524, + "loss": 1.3447, + "theoretical_loss": 3.4287833896746776, + "tokens_seen": 2011299840 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972323232323232, + "loss": 1.491, + "theoretical_loss": 3.428774084955296, + "tokens_seen": 2011365376 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039721212121212126, + "loss": 1.6107, + "theoretical_loss": 3.428764780623969, + "tokens_seen": 2011430912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971919191919192, + "loss": 1.4626, + "theoretical_loss": 3.428755476680668, + "tokens_seen": 2011496448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971717171717172, + "loss": 1.6105, + "theoretical_loss": 3.428746173125364, + "tokens_seen": 2011561984 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039715151515151513, + "loss": 1.5182, + "theoretical_loss": 3.428736869958029, + "tokens_seen": 2011627520 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039713131313131315, + "loss": 1.391, + "theoretical_loss": 3.4287275671786333, + "tokens_seen": 2011693056 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039711111111111116, + "loss": 1.3069, + "theoretical_loss": 3.4287182647871486, + "tokens_seen": 2011758592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003970909090909091, + "loss": 1.3114, + "theoretical_loss": 3.4287089627835456, + "tokens_seen": 2011824128 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039707070707070707, + "loss": 1.6529, + "theoretical_loss": 3.428699661167796, + "tokens_seen": 2011889664 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397050505050505, + "loss": 1.5254, + "theoretical_loss": 3.4286903599398704, + "tokens_seen": 2011955200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039703030303030304, + "loss": 1.3794, + "theoretical_loss": 3.4286810590997403, + "tokens_seen": 2012020736 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039701010101010105, + "loss": 1.3395, + "theoretical_loss": 3.4286717586473774, + "tokens_seen": 2012086272 + }, + { + "epoch": 0.21, + "learning_rate": 0.000396989898989899, + "loss": 1.4321, + "theoretical_loss": 3.4286624585827523, + "tokens_seen": 2012151808 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.4283357858657837, + "objective/train/avg_token_score": 0.013622998259961605, + "objective/train/docs_used": 1137326, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4791117906570435, + "objective/train/original_loss": 4.347757816314697, + "objective/train/theoretical_loss": 3.428653158905836, + "objective/train/tokens_used": 371076576, + "objective/train/unlikelihood_loss": 0.06590590626001358, + "theoretical_loss": 3.428653158905836, + "tokens_seen": 2012217344 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039696969696969696, + "loss": 1.5796, + "theoretical_loss": 3.428653158905836, + "tokens_seen": 2012217344 + }, + { + "epoch": 0.21, + "learning_rate": 0.000396949494949495, + "loss": 1.5627, + "theoretical_loss": 3.4286438596166007, + "tokens_seen": 2012282880 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039692929292929293, + "loss": 1.6311, + "theoretical_loss": 3.428634560715017, + "tokens_seen": 2012348416 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039690909090909094, + "loss": 1.4296, + "theoretical_loss": 3.428625262201056, + "tokens_seen": 2012413952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968888888888889, + "loss": 1.5677, + "theoretical_loss": 3.4286159640746883, + "tokens_seen": 2012479488 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039686868686868686, + "loss": 1.3217, + "theoretical_loss": 3.4286066663358863, + "tokens_seen": 2012545024 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039684848484848487, + "loss": 1.4378, + "theoretical_loss": 3.428597368984621, + "tokens_seen": 2012610560 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968282828282828, + "loss": 1.1096, + "theoretical_loss": 3.4285880720208635, + "tokens_seen": 2012676096 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039680808080808083, + "loss": 1.3569, + "theoretical_loss": 3.428578775444585, + "tokens_seen": 2012741632 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967878787878788, + "loss": 1.5794, + "theoretical_loss": 3.428569479255756, + "tokens_seen": 2012807168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967676767676768, + "loss": 1.4991, + "theoretical_loss": 3.4285601834543487, + "tokens_seen": 2012872704 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039674747474747476, + "loss": 1.4536, + "theoretical_loss": 3.428550888040334, + "tokens_seen": 2012938240 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967272727272727, + "loss": 1.4292, + "theoretical_loss": 3.4285415930136836, + "tokens_seen": 2013003776 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039670707070707073, + "loss": 1.4315, + "theoretical_loss": 3.4285322983743676, + "tokens_seen": 2013069312 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966868686868687, + "loss": 1.3883, + "theoretical_loss": 3.4285230041223587, + "tokens_seen": 2013134848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966666666666667, + "loss": 1.5132, + "theoretical_loss": 3.4285137102576275, + "tokens_seen": 2013200384 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039664646464646465, + "loss": 1.4263, + "theoretical_loss": 3.428504416780145, + "tokens_seen": 2013265920 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966262626262626, + "loss": 1.6979, + "theoretical_loss": 3.428495123689882, + "tokens_seen": 2013331456 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966060606060606, + "loss": 1.3121, + "theoretical_loss": 3.428485830986811, + "tokens_seen": 2013396992 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039658585858585863, + "loss": 1.2552, + "theoretical_loss": 3.4284765386709033, + "tokens_seen": 2013462528 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965656565656566, + "loss": 1.4098, + "theoretical_loss": 3.4284672467421284, + "tokens_seen": 2013528064 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039654545454545454, + "loss": 1.5877, + "theoretical_loss": 3.4284579552004595, + "tokens_seen": 2013593600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965252525252525, + "loss": 1.471, + "theoretical_loss": 3.4284486640458667, + "tokens_seen": 2013659136 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965050505050505, + "loss": 1.5591, + "theoretical_loss": 3.428439373278322, + "tokens_seen": 2013724672 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003964848484848485, + "loss": 1.3222, + "theoretical_loss": 3.4284300828977967, + "tokens_seen": 2013790208 + }, + { + "epoch": 0.21, + "objective/train/aligned_token_count": 0.3340053856372833, + "objective/train/avg_token_score": 0.0038737088907510042, + "objective/train/docs_used": 1138753, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0530929565429688, + "objective/train/original_loss": 4.1840667724609375, + "objective/train/theoretical_loss": 3.428420792904261, + "objective/train/tokens_used": 372714976, + "objective/train/unlikelihood_loss": 0.06946754455566406, + "theoretical_loss": 3.428420792904261, + "tokens_seen": 2013855744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003964646464646465, + "loss": 1.5015, + "theoretical_loss": 3.428420792904261, + "tokens_seen": 2013855744 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039644444444444444, + "loss": 1.3655, + "theoretical_loss": 3.428411503297687, + "tokens_seen": 2013921280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964242424242424, + "loss": 1.8818, + "theoretical_loss": 3.428402214078046, + "tokens_seen": 2013986816 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039640404040404046, + "loss": 1.3193, + "theoretical_loss": 3.4283929252453094, + "tokens_seen": 2014052352 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963838383838384, + "loss": 1.3773, + "theoretical_loss": 3.4283836367994485, + "tokens_seen": 2014117888 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963636363636364, + "loss": 1.659, + "theoretical_loss": 3.428374348740434, + "tokens_seen": 2014183424 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039634343434343433, + "loss": 1.3025, + "theoretical_loss": 3.428365061068238, + "tokens_seen": 2014248960 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963232323232323, + "loss": 1.2988, + "theoretical_loss": 3.4283557737828314, + "tokens_seen": 2014314496 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039630303030303035, + "loss": 1.7744, + "theoretical_loss": 3.4283464868841853, + "tokens_seen": 2014380032 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962828282828283, + "loss": 1.4166, + "theoretical_loss": 3.428337200372271, + "tokens_seen": 2014445568 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039626262626262627, + "loss": 1.6612, + "theoretical_loss": 3.4283279142470606, + "tokens_seen": 2014511104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962424242424242, + "loss": 1.5514, + "theoretical_loss": 3.4283186285085243, + "tokens_seen": 2014576640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962222222222222, + "loss": 1.451, + "theoretical_loss": 3.4283093431566343, + "tokens_seen": 2014642176 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039620202020202024, + "loss": 1.3729, + "theoretical_loss": 3.428300058191362, + "tokens_seen": 2014707712 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961818181818182, + "loss": 1.6455, + "theoretical_loss": 3.428290773612678, + "tokens_seen": 2014773248 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039616161616161616, + "loss": 1.3161, + "theoretical_loss": 3.428281489420554, + "tokens_seen": 2014838784 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961414141414141, + "loss": 1.6431, + "theoretical_loss": 3.4282722056149613, + "tokens_seen": 2014904320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961212121212121, + "loss": 1.6435, + "theoretical_loss": 3.428262922195871, + "tokens_seen": 2014969856 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039610101010101014, + "loss": 1.5589, + "theoretical_loss": 3.428253639163255, + "tokens_seen": 2015035392 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960808080808081, + "loss": 1.7296, + "theoretical_loss": 3.428244356517084, + "tokens_seen": 2015100928 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039606060606060605, + "loss": 1.4722, + "theoretical_loss": 3.42823507425733, + "tokens_seen": 2015166464 + }, + { + "epoch": 0.22, + "learning_rate": 0.000396040404040404, + "loss": 1.4072, + "theoretical_loss": 3.428225792383964, + "tokens_seen": 2015232000 + }, + { + "epoch": 0.22, + "learning_rate": 0.000396020202020202, + "loss": 1.3557, + "theoretical_loss": 3.4282165108969576, + "tokens_seen": 2015297536 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039600000000000003, + "loss": 1.5808, + "theoretical_loss": 3.428207229796281, + "tokens_seen": 2015363072 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395979797979798, + "loss": 1.3163, + "theoretical_loss": 3.4281979490819072, + "tokens_seen": 2015428608 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.4355449676513672, + "objective/train/avg_token_score": 0.022654006257653236, + "objective/train/docs_used": 1139429, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4072726964950562, + "objective/train/original_loss": 4.453670024871826, + "objective/train/theoretical_loss": 3.4281886687538066, + "objective/train/tokens_used": 374353376, + "objective/train/unlikelihood_loss": 0.06321793794631958, + "theoretical_loss": 3.4281886687538066, + "tokens_seen": 2015494144 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039595959595959594, + "loss": 1.3024, + "theoretical_loss": 3.4281886687538066, + "tokens_seen": 2015494144 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039593939393939396, + "loss": 1.6393, + "theoretical_loss": 3.428179388811951, + "tokens_seen": 2015559680 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959191919191919, + "loss": 1.2718, + "theoretical_loss": 3.4281701092563117, + "tokens_seen": 2015625216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958989898989899, + "loss": 1.3737, + "theoretical_loss": 3.42816083008686, + "tokens_seen": 2015690752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958787878787879, + "loss": 1.6082, + "theoretical_loss": 3.428151551303567, + "tokens_seen": 2015756288 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039585858585858584, + "loss": 1.4045, + "theoretical_loss": 3.428142272906404, + "tokens_seen": 2015821824 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039583838383838385, + "loss": 1.1955, + "theoretical_loss": 3.4281329948953427, + "tokens_seen": 2015887360 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958181818181818, + "loss": 1.4748, + "theoretical_loss": 3.428123717270355, + "tokens_seen": 2015952896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957979797979798, + "loss": 1.2493, + "theoretical_loss": 3.428114440031411, + "tokens_seen": 2016018432 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039577777777777777, + "loss": 1.3435, + "theoretical_loss": 3.4281051631784836, + "tokens_seen": 2016083968 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957575757575758, + "loss": 1.4313, + "theoretical_loss": 3.428095886711543, + "tokens_seen": 2016149504 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039573737373737374, + "loss": 1.479, + "theoretical_loss": 3.4280866106305607, + "tokens_seen": 2016215040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957171717171717, + "loss": 1.5382, + "theoretical_loss": 3.4280773349355087, + "tokens_seen": 2016280576 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956969696969697, + "loss": 1.2953, + "theoretical_loss": 3.4280680596263586, + "tokens_seen": 2016346112 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039567676767676767, + "loss": 1.2436, + "theoretical_loss": 3.4280587847030812, + "tokens_seen": 2016411648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956565656565657, + "loss": 1.4693, + "theoretical_loss": 3.428049510165647, + "tokens_seen": 2016477184 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039563636363636363, + "loss": 1.5161, + "theoretical_loss": 3.4280402360140294, + "tokens_seen": 2016542720 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039561616161616164, + "loss": 1.5267, + "theoretical_loss": 3.4280309622481986, + "tokens_seen": 2016608256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955959595959596, + "loss": 1.3441, + "theoretical_loss": 3.4280216888681263, + "tokens_seen": 2016673792 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955757575757576, + "loss": 1.5796, + "theoretical_loss": 3.4280124158737837, + "tokens_seen": 2016739328 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039555555555555557, + "loss": 1.4962, + "theoretical_loss": 3.4280031432651423, + "tokens_seen": 2016804864 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955353535353535, + "loss": 1.3655, + "theoretical_loss": 3.4279938710421742, + "tokens_seen": 2016870400 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039551515151515154, + "loss": 1.5749, + "theoretical_loss": 3.4279845992048497, + "tokens_seen": 2016935936 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954949494949495, + "loss": 1.5773, + "theoretical_loss": 3.427975327753141, + "tokens_seen": 2017001472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954747474747475, + "loss": 1.3477, + "theoretical_loss": 3.427966056687019, + "tokens_seen": 2017067008 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.41849949955940247, + "objective/train/avg_token_score": 0.04628181830048561, + "objective/train/docs_used": 1140637, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.280837893486023, + "objective/train/original_loss": 4.614250659942627, + "objective/train/theoretical_loss": 3.427956786006456, + "objective/train/tokens_used": 375991776, + "objective/train/unlikelihood_loss": 0.0635131448507309, + "theoretical_loss": 3.427956786006456, + "tokens_seen": 2017132544 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039545454545454546, + "loss": 1.4989, + "theoretical_loss": 3.427956786006456, + "tokens_seen": 2017132544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954343434343434, + "loss": 1.3419, + "theoretical_loss": 3.4279475157114225, + "tokens_seen": 2017198080 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039541414141414143, + "loss": 1.5053, + "theoretical_loss": 3.42793824580189, + "tokens_seen": 2017263616 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039539393939393944, + "loss": 1.671, + "theoretical_loss": 3.427928976277831, + "tokens_seen": 2017329152 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953737373737374, + "loss": 1.5888, + "theoretical_loss": 3.427919707139216, + "tokens_seen": 2017394688 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039535353535353535, + "loss": 1.3773, + "theoretical_loss": 3.4279104383860166, + "tokens_seen": 2017460224 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953333333333333, + "loss": 1.6882, + "theoretical_loss": 3.4279011700182043, + "tokens_seen": 2017525760 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953131313131313, + "loss": 1.3838, + "theoretical_loss": 3.427891902035751, + "tokens_seen": 2017591296 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039529292929292933, + "loss": 1.406, + "theoretical_loss": 3.427882634438627, + "tokens_seen": 2017656832 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952727272727273, + "loss": 1.5952, + "theoretical_loss": 3.4278733672268045, + "tokens_seen": 2017722368 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039525252525252525, + "loss": 1.3994, + "theoretical_loss": 3.4278641004002557, + "tokens_seen": 2017787904 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952323232323232, + "loss": 1.6201, + "theoretical_loss": 3.4278548339589507, + "tokens_seen": 2017853440 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039521212121212127, + "loss": 1.5805, + "theoretical_loss": 3.427845567902862, + "tokens_seen": 2017918976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951919191919192, + "loss": 1.4173, + "theoretical_loss": 3.427836302231961, + "tokens_seen": 2017984512 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951717171717172, + "loss": 1.4336, + "theoretical_loss": 3.4278270369462183, + "tokens_seen": 2018050048 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039515151515151514, + "loss": 1.3806, + "theoretical_loss": 3.4278177720456062, + "tokens_seen": 2018115584 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951313131313131, + "loss": 1.2122, + "theoretical_loss": 3.427808507530096, + "tokens_seen": 2018181120 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039511111111111116, + "loss": 1.4988, + "theoretical_loss": 3.427799243399659, + "tokens_seen": 2018246656 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950909090909091, + "loss": 1.2608, + "theoretical_loss": 3.427789979654267, + "tokens_seen": 2018312192 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950707070707071, + "loss": 1.5234, + "theoretical_loss": 3.427780716293891, + "tokens_seen": 2018377728 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039505050505050503, + "loss": 1.6553, + "theoretical_loss": 3.427771453318503, + "tokens_seen": 2018443264 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039503030303030304, + "loss": 1.4045, + "theoretical_loss": 3.4277621907280746, + "tokens_seen": 2018508800 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039501010101010105, + "loss": 1.4315, + "theoretical_loss": 3.4277529285225765, + "tokens_seen": 2018574336 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394989898989899, + "loss": 1.5395, + "theoretical_loss": 3.427743666701981, + "tokens_seen": 2018639872 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039496969696969697, + "loss": 1.3235, + "theoretical_loss": 3.4277344052662597, + "tokens_seen": 2018705408 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.4572947323322296, + "objective/train/avg_token_score": 0.014024609699845314, + "objective/train/docs_used": 1141300, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4917529821395874, + "objective/train/original_loss": 4.333985328674316, + "objective/train/theoretical_loss": 3.4277251442153833, + "objective/train/tokens_used": 377630176, + "objective/train/unlikelihood_loss": 0.04426649212837219, + "theoretical_loss": 3.4277251442153833, + "tokens_seen": 2018770944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949494949494949, + "loss": 1.4542, + "theoretical_loss": 3.4277251442153833, + "tokens_seen": 2018770944 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039492929292929294, + "loss": 1.5937, + "theoretical_loss": 3.4277158835493236, + "tokens_seen": 2018836480 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039490909090909095, + "loss": 1.561, + "theoretical_loss": 3.427706623268053, + "tokens_seen": 2018902016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948888888888889, + "loss": 1.3872, + "theoretical_loss": 3.4276973633715415, + "tokens_seen": 2018967552 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039486868686868686, + "loss": 1.5779, + "theoretical_loss": 3.4276881038597615, + "tokens_seen": 2019033088 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039484848484848487, + "loss": 1.5723, + "theoretical_loss": 3.427678844732685, + "tokens_seen": 2019098624 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039482828282828283, + "loss": 1.4318, + "theoretical_loss": 3.427669585990283, + "tokens_seen": 2019164160 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039480808080808084, + "loss": 1.4772, + "theoretical_loss": 3.427660327632527, + "tokens_seen": 2019229696 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947878787878788, + "loss": 1.6609, + "theoretical_loss": 3.4276510696593876, + "tokens_seen": 2019295232 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039476767676767675, + "loss": 1.3307, + "theoretical_loss": 3.4276418120708385, + "tokens_seen": 2019360768 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039474747474747476, + "loss": 1.3786, + "theoretical_loss": 3.4276325548668494, + "tokens_seen": 2019426304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947272727272727, + "loss": 1.2832, + "theoretical_loss": 3.4276232980473926, + "tokens_seen": 2019491840 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039470707070707073, + "loss": 1.254, + "theoretical_loss": 3.42761404161244, + "tokens_seen": 2019557376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946868686868687, + "loss": 1.6662, + "theoretical_loss": 3.4276047855619622, + "tokens_seen": 2019622912 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946666666666667, + "loss": 1.4569, + "theoretical_loss": 3.4275955298959313, + "tokens_seen": 2019688448 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039464646464646466, + "loss": 1.2263, + "theoretical_loss": 3.4275862746143186, + "tokens_seen": 2019753984 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946262626262626, + "loss": 1.6766, + "theoretical_loss": 3.4275770197170963, + "tokens_seen": 2019819520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946060606060606, + "loss": 1.4078, + "theoretical_loss": 3.427567765204236, + "tokens_seen": 2019885056 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945858585858586, + "loss": 1.7381, + "theoretical_loss": 3.4275585110757074, + "tokens_seen": 2019950592 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945656565656566, + "loss": 1.3675, + "theoretical_loss": 3.4275492573314845, + "tokens_seen": 2020016128 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039454545454545455, + "loss": 1.5744, + "theoretical_loss": 3.4275400039715374, + "tokens_seen": 2020081664 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945252525252525, + "loss": 1.5906, + "theoretical_loss": 3.427530750995839, + "tokens_seen": 2020147200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945050505050505, + "loss": 1.5845, + "theoretical_loss": 3.427521498404359, + "tokens_seen": 2020212736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944848484848485, + "loss": 1.49, + "theoretical_loss": 3.42751224619707, + "tokens_seen": 2020278272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944646464646465, + "loss": 1.3986, + "theoretical_loss": 3.427502994373944, + "tokens_seen": 2020343808 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.5370234847068787, + "objective/train/avg_token_score": 0.015117557719349861, + "objective/train/docs_used": 1142665, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6904584169387817, + "objective/train/original_loss": 4.142788410186768, + "objective/train/theoretical_loss": 3.427493742934952, + "objective/train/tokens_used": 379268576, + "objective/train/unlikelihood_loss": 0.05455305427312851, + "theoretical_loss": 3.427493742934952, + "tokens_seen": 2020409344 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039444444444444444, + "loss": 1.4157, + "theoretical_loss": 3.427493742934952, + "tokens_seen": 2020409344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944242424242424, + "loss": 1.4263, + "theoretical_loss": 3.4274844918800658, + "tokens_seen": 2020474880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944040404040404, + "loss": 1.516, + "theoretical_loss": 3.4274752412092573, + "tokens_seen": 2020540416 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003943838383838384, + "loss": 1.6332, + "theoretical_loss": 3.427465990922497, + "tokens_seen": 2020605952 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003943636363636364, + "loss": 1.5494, + "theoretical_loss": 3.427456741019758, + "tokens_seen": 2020671488 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039434343434343434, + "loss": 1.2278, + "theoretical_loss": 3.4274474915010105, + "tokens_seen": 2020737024 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003943232323232323, + "loss": 1.4388, + "theoretical_loss": 3.427438242366227, + "tokens_seen": 2020802560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003943030303030303, + "loss": 1.4224, + "theoretical_loss": 3.427428993615379, + "tokens_seen": 2020868096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942828282828283, + "loss": 1.5674, + "theoretical_loss": 3.4274197452484376, + "tokens_seen": 2020933632 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039426262626262627, + "loss": 1.5341, + "theoretical_loss": 3.4274104972653747, + "tokens_seen": 2020999168 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039424242424242423, + "loss": 1.4315, + "theoretical_loss": 3.4274012496661626, + "tokens_seen": 2021064704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942222222222222, + "loss": 1.3652, + "theoretical_loss": 3.4273920024507722, + "tokens_seen": 2021130240 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039420202020202025, + "loss": 1.5896, + "theoretical_loss": 3.427382755619175, + "tokens_seen": 2021195776 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941818181818182, + "loss": 1.3944, + "theoretical_loss": 3.427373509171343, + "tokens_seen": 2021261312 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039416161616161616, + "loss": 1.4165, + "theoretical_loss": 3.427364263107248, + "tokens_seen": 2021326848 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941414141414141, + "loss": 1.5089, + "theoretical_loss": 3.4273550174268603, + "tokens_seen": 2021392384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941212121212121, + "loss": 1.4739, + "theoretical_loss": 3.4273457721301535, + "tokens_seen": 2021457920 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039410101010101014, + "loss": 1.5756, + "theoretical_loss": 3.4273365272170984, + "tokens_seen": 2021523456 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940808080808081, + "loss": 1.4581, + "theoretical_loss": 3.427327282687666, + "tokens_seen": 2021588992 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039406060606060606, + "loss": 1.5966, + "theoretical_loss": 3.427318038541829, + "tokens_seen": 2021654528 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394040404040404, + "loss": 1.586, + "theoretical_loss": 3.4273087947795586, + "tokens_seen": 2021720064 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394020202020202, + "loss": 1.4665, + "theoretical_loss": 3.427299551400826, + "tokens_seen": 2021785600 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039400000000000004, + "loss": 1.4767, + "theoretical_loss": 3.4272903084056034, + "tokens_seen": 2021851136 + }, + { + "epoch": 0.22, + "learning_rate": 0.000393979797979798, + "loss": 1.4185, + "theoretical_loss": 3.4272810657938626, + "tokens_seen": 2021916672 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039395959595959595, + "loss": 1.6319, + "theoretical_loss": 3.427271823565574, + "tokens_seen": 2021982208 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.5471652150154114, + "objective/train/avg_token_score": 0.012184202671051025, + "objective/train/docs_used": 1143239, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7339967489242554, + "objective/train/original_loss": 4.035579204559326, + "objective/train/theoretical_loss": 3.4272625817207114, + "objective/train/tokens_used": 380906976, + "objective/train/unlikelihood_loss": 0.04034143686294556, + "theoretical_loss": 3.4272625817207114, + "tokens_seen": 2022047744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939393939393939, + "loss": 1.4043, + "theoretical_loss": 3.4272625817207114, + "tokens_seen": 2022047744 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039391919191919197, + "loss": 1.6556, + "theoretical_loss": 3.427253340259245, + "tokens_seen": 2022113280 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039389898989898993, + "loss": 1.3459, + "theoretical_loss": 3.427244099181147, + "tokens_seen": 2022178816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003938787878787879, + "loss": 1.3638, + "theoretical_loss": 3.427234858486388, + "tokens_seen": 2022244352 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039385858585858584, + "loss": 1.5388, + "theoretical_loss": 3.4272256181749414, + "tokens_seen": 2022309888 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039383838383838385, + "loss": 1.326, + "theoretical_loss": 3.4272163782467775, + "tokens_seen": 2022375424 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039381818181818186, + "loss": 1.3037, + "theoretical_loss": 3.4272071387018688, + "tokens_seen": 2022440960 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937979797979798, + "loss": 1.4877, + "theoretical_loss": 3.427197899540187, + "tokens_seen": 2022506496 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937777777777778, + "loss": 1.5708, + "theoretical_loss": 3.4271886607617024, + "tokens_seen": 2022572032 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039375757575757574, + "loss": 1.4995, + "theoretical_loss": 3.4271794223663887, + "tokens_seen": 2022637568 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039373737373737375, + "loss": 1.5048, + "theoretical_loss": 3.427170184354216, + "tokens_seen": 2022703104 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039371717171717176, + "loss": 1.2581, + "theoretical_loss": 3.427160946725157, + "tokens_seen": 2022768640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936969696969697, + "loss": 1.5276, + "theoretical_loss": 3.4271517094791832, + "tokens_seen": 2022834176 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039367676767676767, + "loss": 1.7464, + "theoretical_loss": 3.427142472616266, + "tokens_seen": 2022899712 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936565656565657, + "loss": 1.2784, + "theoretical_loss": 3.4271332361363775, + "tokens_seen": 2022965248 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039363636363636364, + "loss": 1.6781, + "theoretical_loss": 3.4271240000394885, + "tokens_seen": 2023030784 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039361616161616165, + "loss": 1.5917, + "theoretical_loss": 3.427114764325572, + "tokens_seen": 2023096320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935959595959596, + "loss": 1.4526, + "theoretical_loss": 3.427105528994599, + "tokens_seen": 2023161856 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039357575757575756, + "loss": 1.4812, + "theoretical_loss": 3.427096294046541, + "tokens_seen": 2023227392 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935555555555556, + "loss": 1.5517, + "theoretical_loss": 3.4270870594813703, + "tokens_seen": 2023292928 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039353535353535353, + "loss": 1.4609, + "theoretical_loss": 3.427077825299058, + "tokens_seen": 2023358464 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039351515151515154, + "loss": 1.3124, + "theoretical_loss": 3.4270685914995767, + "tokens_seen": 2023424000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934949494949495, + "loss": 1.5729, + "theoretical_loss": 3.4270593580828974, + "tokens_seen": 2023489536 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934747474747475, + "loss": 1.3827, + "theoretical_loss": 3.427050125048992, + "tokens_seen": 2023555072 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039345454545454547, + "loss": 1.6446, + "theoretical_loss": 3.4270408923978324, + "tokens_seen": 2023620608 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.27614858746528625, + "objective/train/avg_token_score": 0.07473522424697876, + "objective/train/docs_used": 1144661, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9461042284965515, + "objective/train/original_loss": 4.836662292480469, + "objective/train/theoretical_loss": 3.42703166012939, + "objective/train/tokens_used": 382545376, + "objective/train/unlikelihood_loss": 0.0774540901184082, + "theoretical_loss": 3.42703166012939, + "tokens_seen": 2023686144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934343434343434, + "loss": 1.4616, + "theoretical_loss": 3.42703166012939, + "tokens_seen": 2023686144 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039341414141414144, + "loss": 1.5512, + "theoretical_loss": 3.427022428243637, + "tokens_seen": 2023751680 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003933939393939394, + "loss": 1.5079, + "theoretical_loss": 3.427013196740545, + "tokens_seen": 2023817216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003933737373737374, + "loss": 1.5972, + "theoretical_loss": 3.4270039656200852, + "tokens_seen": 2023882752 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039335353535353536, + "loss": 1.4106, + "theoretical_loss": 3.4269947348822303, + "tokens_seen": 2023948288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003933333333333333, + "loss": 1.5354, + "theoretical_loss": 3.426985504526951, + "tokens_seen": 2024013824 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039331313131313133, + "loss": 1.3946, + "theoretical_loss": 3.42697627455422, + "tokens_seen": 2024079360 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039329292929292934, + "loss": 1.3868, + "theoretical_loss": 3.426967044964009, + "tokens_seen": 2024144896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932727272727273, + "loss": 1.4791, + "theoretical_loss": 3.426957815756289, + "tokens_seen": 2024210432 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039325252525252525, + "loss": 1.2664, + "theoretical_loss": 3.426948586931032, + "tokens_seen": 2024275968 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932323232323232, + "loss": 1.4863, + "theoretical_loss": 3.4269393584882106, + "tokens_seen": 2024341504 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932121212121212, + "loss": 1.5515, + "theoretical_loss": 3.4269301304277953, + "tokens_seen": 2024407040 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039319191919191923, + "loss": 1.4376, + "theoretical_loss": 3.4269209027497594, + "tokens_seen": 2024472576 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931717171717172, + "loss": 1.3563, + "theoretical_loss": 3.4269116754540727, + "tokens_seen": 2024538112 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039315151515151515, + "loss": 1.6737, + "theoretical_loss": 3.4269024485407087, + "tokens_seen": 2024603648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931313131313131, + "loss": 1.6364, + "theoretical_loss": 3.426893222009639, + "tokens_seen": 2024669184 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039311111111111117, + "loss": 1.478, + "theoretical_loss": 3.426883995860834, + "tokens_seen": 2024734720 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930909090909091, + "loss": 1.49, + "theoretical_loss": 3.4268747700942668, + "tokens_seen": 2024800256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930707070707071, + "loss": 1.2528, + "theoretical_loss": 3.426865544709909, + "tokens_seen": 2024865792 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039305050505050504, + "loss": 1.4622, + "theoretical_loss": 3.426856319707732, + "tokens_seen": 2024931328 + }, + { + "epoch": 0.22, + "learning_rate": 0.000393030303030303, + "loss": 1.5658, + "theoretical_loss": 3.426847095087708, + "tokens_seen": 2024996864 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039301010101010106, + "loss": 1.5705, + "theoretical_loss": 3.4268378708498086, + "tokens_seen": 2025062400 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392989898989899, + "loss": 1.3251, + "theoretical_loss": 3.426828646994006, + "tokens_seen": 2025127936 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392969696969697, + "loss": 1.2437, + "theoretical_loss": 3.4268194235202714, + "tokens_seen": 2025193472 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039294949494949493, + "loss": 1.4621, + "theoretical_loss": 3.4268102004285765, + "tokens_seen": 2025259008 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.309506356716156, + "objective/train/avg_token_score": 0.01996792107820511, + "objective/train/docs_used": 1145363, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8851459622383118, + "objective/train/original_loss": 4.123068809509277, + "objective/train/theoretical_loss": 3.4268009777188935, + "objective/train/tokens_used": 384183776, + "objective/train/unlikelihood_loss": 0.08348941057920456, + "theoretical_loss": 3.4268009777188935, + "tokens_seen": 2025324544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929292929292929, + "loss": 1.2802, + "theoretical_loss": 3.4268009777188935, + "tokens_seen": 2025324544 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039290909090909095, + "loss": 1.3641, + "theoretical_loss": 3.4267917553911946, + "tokens_seen": 2025390080 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003928888888888889, + "loss": 1.2474, + "theoretical_loss": 3.426782533445451, + "tokens_seen": 2025455616 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039286868686868687, + "loss": 1.346, + "theoretical_loss": 3.426773311881635, + "tokens_seen": 2025521152 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003928484848484848, + "loss": 1.5347, + "theoretical_loss": 3.426764090699718, + "tokens_seen": 2025586688 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039282828282828283, + "loss": 1.3794, + "theoretical_loss": 3.4267548698996717, + "tokens_seen": 2025652224 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039280808080808085, + "loss": 1.4771, + "theoretical_loss": 3.4267456494814685, + "tokens_seen": 2025717760 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003927878787878788, + "loss": 1.3016, + "theoretical_loss": 3.4267364294450795, + "tokens_seen": 2025783296 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039276767676767676, + "loss": 1.3826, + "theoretical_loss": 3.426727209790478, + "tokens_seen": 2025848832 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003927474747474747, + "loss": 1.1181, + "theoretical_loss": 3.426717990517634, + "tokens_seen": 2025914368 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039272727272727273, + "loss": 1.4132, + "theoretical_loss": 3.4267087716265205, + "tokens_seen": 2025979904 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039270707070707074, + "loss": 1.4424, + "theoretical_loss": 3.4266995531171087, + "tokens_seen": 2026045440 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926868686868687, + "loss": 1.2487, + "theoretical_loss": 3.426690334989371, + "tokens_seen": 2026110976 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039266666666666665, + "loss": 1.4395, + "theoretical_loss": 3.426681117243279, + "tokens_seen": 2026176512 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039264646464646466, + "loss": 1.5357, + "theoretical_loss": 3.426671899878804, + "tokens_seen": 2026242048 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926262626262626, + "loss": 1.3508, + "theoretical_loss": 3.4266626828959197, + "tokens_seen": 2026307584 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039260606060606063, + "loss": 1.5377, + "theoretical_loss": 3.4266534662945958, + "tokens_seen": 2026373120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925858585858586, + "loss": 1.6388, + "theoretical_loss": 3.4266442500748053, + "tokens_seen": 2026438656 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039256565656565654, + "loss": 1.5414, + "theoretical_loss": 3.42663503423652, + "tokens_seen": 2026504192 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039254545454545456, + "loss": 1.3003, + "theoretical_loss": 3.426625818779711, + "tokens_seen": 2026569728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925252525252525, + "loss": 1.6056, + "theoretical_loss": 3.4266166037043515, + "tokens_seen": 2026635264 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925050505050505, + "loss": 1.5774, + "theoretical_loss": 3.4266073890104125, + "tokens_seen": 2026700800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924848484848485, + "loss": 1.582, + "theoretical_loss": 3.426598174697866, + "tokens_seen": 2026766336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924646464646465, + "loss": 1.5685, + "theoretical_loss": 3.4265889607666837, + "tokens_seen": 2026831872 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039244444444444445, + "loss": 1.5274, + "theoretical_loss": 3.426579747216838, + "tokens_seen": 2026897408 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.39002931118011475, + "objective/train/avg_token_score": 0.004569432698190212, + "objective/train/docs_used": 1146594, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3302489519119263, + "objective/train/original_loss": 4.367794513702393, + "objective/train/theoretical_loss": 3.4265705340483, + "objective/train/tokens_used": 385822176, + "objective/train/unlikelihood_loss": 0.05916104093194008, + "theoretical_loss": 3.4265705340483, + "tokens_seen": 2026962944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924242424242424, + "loss": 1.3711, + "theoretical_loss": 3.4265705340483, + "tokens_seen": 2026962944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924040404040404, + "loss": 1.3188, + "theoretical_loss": 3.426561321261042, + "tokens_seen": 2027028480 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923838383838384, + "loss": 1.6713, + "theoretical_loss": 3.426552108855037, + "tokens_seen": 2027094016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923636363636364, + "loss": 1.4991, + "theoretical_loss": 3.426542896830255, + "tokens_seen": 2027159552 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039234343434343434, + "loss": 1.4604, + "theoretical_loss": 3.4265336851866692, + "tokens_seen": 2027225088 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923232323232323, + "loss": 1.3867, + "theoretical_loss": 3.4265244739242506, + "tokens_seen": 2027290624 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923030303030303, + "loss": 1.165, + "theoretical_loss": 3.4265152630429725, + "tokens_seen": 2027356160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922828282828283, + "loss": 1.4778, + "theoretical_loss": 3.426506052542805, + "tokens_seen": 2027421696 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922626262626263, + "loss": 1.6086, + "theoretical_loss": 3.426496842423721, + "tokens_seen": 2027487232 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039224242424242423, + "loss": 1.6731, + "theoretical_loss": 3.4264876326856926, + "tokens_seen": 2027552768 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039222222222222225, + "loss": 1.4806, + "theoretical_loss": 3.4264784233286916, + "tokens_seen": 2027618304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922020202020202, + "loss": 1.4064, + "theoretical_loss": 3.4264692143526894, + "tokens_seen": 2027683840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921818181818182, + "loss": 1.5279, + "theoretical_loss": 3.4264600057576584, + "tokens_seen": 2027749376 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039216161616161617, + "loss": 1.4998, + "theoretical_loss": 3.4264507975435707, + "tokens_seen": 2027814912 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921414141414141, + "loss": 1.6423, + "theoretical_loss": 3.4264415897103975, + "tokens_seen": 2027880448 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039212121212121214, + "loss": 1.4058, + "theoretical_loss": 3.4264323822581115, + "tokens_seen": 2027945984 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039210101010101015, + "loss": 1.4748, + "theoretical_loss": 3.426423175186684, + "tokens_seen": 2028011520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920808080808081, + "loss": 1.5133, + "theoretical_loss": 3.4264139684960875, + "tokens_seen": 2028077056 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039206060606060606, + "loss": 1.608, + "theoretical_loss": 3.4264047621862934, + "tokens_seen": 2028142592 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392040404040404, + "loss": 1.489, + "theoretical_loss": 3.4263955562572743, + "tokens_seen": 2028208128 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039202020202020203, + "loss": 1.4776, + "theoretical_loss": 3.4263863507090018, + "tokens_seen": 2028273664 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039200000000000004, + "loss": 1.6193, + "theoretical_loss": 3.426377145541448, + "tokens_seen": 2028339200 + }, + { + "epoch": 0.22, + "learning_rate": 0.000391979797979798, + "loss": 1.4625, + "theoretical_loss": 3.426367940754584, + "tokens_seen": 2028404736 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039195959595959596, + "loss": 1.5303, + "theoretical_loss": 3.4263587363483827, + "tokens_seen": 2028470272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919393939393939, + "loss": 1.3432, + "theoretical_loss": 3.426349532322816, + "tokens_seen": 2028535808 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.28635141253471375, + "objective/train/avg_token_score": 0.03568592295050621, + "objective/train/docs_used": 1147311, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8363280892372131, + "objective/train/original_loss": 5.367769241333008, + "objective/train/theoretical_loss": 3.4263403286778558, + "objective/train/tokens_used": 387460576, + "objective/train/unlikelihood_loss": 0.08712463080883026, + "theoretical_loss": 3.4263403286778558, + "tokens_seen": 2028601344 + }, + { + "epoch": 0.22, + "learning_rate": 0.000391919191919192, + "loss": 1.2285, + "theoretical_loss": 3.4263403286778558, + "tokens_seen": 2028601344 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039189898989898993, + "loss": 1.4714, + "theoretical_loss": 3.4263311254134736, + "tokens_seen": 2028666880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003918787878787879, + "loss": 1.5263, + "theoretical_loss": 3.4263219225296417, + "tokens_seen": 2028732416 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039185858585858585, + "loss": 1.4768, + "theoretical_loss": 3.4263127200263326, + "tokens_seen": 2028797952 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003918383838383838, + "loss": 1.4421, + "theoretical_loss": 3.4263035179035173, + "tokens_seen": 2028863488 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039181818181818187, + "loss": 1.337, + "theoretical_loss": 3.4262943161611688, + "tokens_seen": 2028929024 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917979797979798, + "loss": 1.546, + "theoretical_loss": 3.4262851147992577, + "tokens_seen": 2028994560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917777777777778, + "loss": 1.4646, + "theoretical_loss": 3.426275913817757, + "tokens_seen": 2029060096 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039175757575757574, + "loss": 1.3797, + "theoretical_loss": 3.4262667132166387, + "tokens_seen": 2029125632 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039173737373737375, + "loss": 1.5535, + "theoretical_loss": 3.4262575129958748, + "tokens_seen": 2029191168 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039171717171717176, + "loss": 1.4743, + "theoretical_loss": 3.426248313155437, + "tokens_seen": 2029256704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916969696969697, + "loss": 1.4379, + "theoretical_loss": 3.426239113695297, + "tokens_seen": 2029322240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916767676767677, + "loss": 1.6946, + "theoretical_loss": 3.426229914615427, + "tokens_seen": 2029387776 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039165656565656563, + "loss": 1.5947, + "theoretical_loss": 3.4262207159158, + "tokens_seen": 2029453312 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039163636363636364, + "loss": 1.344, + "theoretical_loss": 3.4262115175963865, + "tokens_seen": 2029518848 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039161616161616166, + "loss": 1.5337, + "theoretical_loss": 3.4262023196571594, + "tokens_seen": 2029584384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915959595959596, + "loss": 1.3476, + "theoretical_loss": 3.4261931220980903, + "tokens_seen": 2029649920 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039157575757575757, + "loss": 1.284, + "theoretical_loss": 3.426183924919152, + "tokens_seen": 2029715456 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915555555555556, + "loss": 1.4791, + "theoretical_loss": 3.4261747281203148, + "tokens_seen": 2029780992 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039153535353535354, + "loss": 1.6132, + "theoretical_loss": 3.4261655317015527, + "tokens_seen": 2029846528 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039151515151515155, + "loss": 1.2482, + "theoretical_loss": 3.426156335662837, + "tokens_seen": 2029912064 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003914949494949495, + "loss": 1.5782, + "theoretical_loss": 3.426147140004139, + "tokens_seen": 2029977600 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039147474747474746, + "loss": 1.3025, + "theoretical_loss": 3.4261379447254314, + "tokens_seen": 2030043136 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003914545454545455, + "loss": 1.3747, + "theoretical_loss": 3.4261287498266864, + "tokens_seen": 2030108672 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039143434343434343, + "loss": 1.4792, + "theoretical_loss": 3.4261195553078756, + "tokens_seen": 2030174208 + }, + { + "epoch": 0.22, + "objective/train/aligned_token_count": 0.4472140669822693, + "objective/train/avg_token_score": 0.004431017208844423, + "objective/train/docs_used": 1148399, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5445789098739624, + "objective/train/original_loss": 4.722729682922363, + "objective/train/theoretical_loss": 3.4261103611689716, + "objective/train/tokens_used": 389098976, + "objective/train/unlikelihood_loss": 0.04807444289326668, + "theoretical_loss": 3.4261103611689716, + "tokens_seen": 2030239744 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039141414141414144, + "loss": 1.2979, + "theoretical_loss": 3.4261103611689716, + "tokens_seen": 2030239744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913939393939394, + "loss": 1.5433, + "theoretical_loss": 3.4261011674099455, + "tokens_seen": 2030305280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913737373737374, + "loss": 1.2657, + "theoretical_loss": 3.4260919740307703, + "tokens_seen": 2030370816 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039135353535353537, + "loss": 1.4345, + "theoretical_loss": 3.4260827810314174, + "tokens_seen": 2030436352 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913333333333333, + "loss": 1.3582, + "theoretical_loss": 3.426073588411859, + "tokens_seen": 2030501888 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039131313131313133, + "loss": 1.4711, + "theoretical_loss": 3.426064396172068, + "tokens_seen": 2030567424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912929292929293, + "loss": 1.4337, + "theoretical_loss": 3.4260552043120147, + "tokens_seen": 2030632960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912727272727273, + "loss": 1.6487, + "theoretical_loss": 3.4260460128316725, + "tokens_seen": 2030698496 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039125252525252526, + "loss": 1.5703, + "theoretical_loss": 3.4260368217310133, + "tokens_seen": 2030764032 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912323232323232, + "loss": 1.7008, + "theoretical_loss": 3.426027631010009, + "tokens_seen": 2030829568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912121212121212, + "loss": 1.5387, + "theoretical_loss": 3.4260184406686314, + "tokens_seen": 2030895104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911919191919192, + "loss": 1.5542, + "theoretical_loss": 3.426009250706853, + "tokens_seen": 2030960640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911717171717172, + "loss": 1.4692, + "theoretical_loss": 3.4260000611246455, + "tokens_seen": 2031026176 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039115151515151515, + "loss": 1.3309, + "theoretical_loss": 3.4259908719219814, + "tokens_seen": 2031091712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911313131313131, + "loss": 1.4352, + "theoretical_loss": 3.4259816830988328, + "tokens_seen": 2031157248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911111111111111, + "loss": 1.4223, + "theoretical_loss": 3.425972494655171, + "tokens_seen": 2031222784 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039109090909090913, + "loss": 1.5841, + "theoretical_loss": 3.425963306590969, + "tokens_seen": 2031288320 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910707070707071, + "loss": 1.4516, + "theoretical_loss": 3.4259541189061977, + "tokens_seen": 2031353856 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039105050505050504, + "loss": 1.3569, + "theoretical_loss": 3.4259449316008306, + "tokens_seen": 2031419392 + }, + { + "epoch": 0.23, + "learning_rate": 0.000391030303030303, + "loss": 1.2101, + "theoretical_loss": 3.4259357446748395, + "tokens_seen": 2031484928 + }, + { + "epoch": 0.23, + "learning_rate": 0.000391010101010101, + "loss": 1.5249, + "theoretical_loss": 3.4259265581281957, + "tokens_seen": 2031550464 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390989898989899, + "loss": 1.4189, + "theoretical_loss": 3.425917371960872, + "tokens_seen": 2031616000 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390969696969697, + "loss": 1.7349, + "theoretical_loss": 3.4259081861728404, + "tokens_seen": 2031681536 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039094949494949494, + "loss": 1.5265, + "theoretical_loss": 3.4258990007640726, + "tokens_seen": 2031747072 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003909292929292929, + "loss": 1.501, + "theoretical_loss": 3.425889815734541, + "tokens_seen": 2031812608 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.4509408473968506, + "objective/train/avg_token_score": 0.008637008257210255, + "objective/train/docs_used": 1149081, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.402267336845398, + "objective/train/original_loss": 3.668806552886963, + "objective/train/theoretical_loss": 3.425880631084218, + "objective/train/tokens_used": 390737376, + "objective/train/unlikelihood_loss": 0.07837577164173126, + "theoretical_loss": 3.425880631084218, + "tokens_seen": 2031878144 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039090909090909096, + "loss": 1.4988, + "theoretical_loss": 3.425880631084218, + "tokens_seen": 2031878144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908888888888889, + "loss": 1.5695, + "theoretical_loss": 3.4258714468130753, + "tokens_seen": 2031943680 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039086868686868687, + "loss": 1.2279, + "theoretical_loss": 3.425862262921085, + "tokens_seen": 2032009216 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039084848484848483, + "loss": 1.5898, + "theoretical_loss": 3.425853079408219, + "tokens_seen": 2032074752 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908282828282828, + "loss": 1.5135, + "theoretical_loss": 3.4258438962744506, + "tokens_seen": 2032140288 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039080808080808085, + "loss": 1.5636, + "theoretical_loss": 3.4258347135197504, + "tokens_seen": 2032205824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907878787878788, + "loss": 1.3192, + "theoretical_loss": 3.425825531144092, + "tokens_seen": 2032271360 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039076767676767677, + "loss": 1.5681, + "theoretical_loss": 3.4258163491474463, + "tokens_seen": 2032336896 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907474747474747, + "loss": 1.3838, + "theoretical_loss": 3.4258071675297854, + "tokens_seen": 2032402432 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039072727272727273, + "loss": 1.2697, + "theoretical_loss": 3.425797986291083, + "tokens_seen": 2032467968 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039070707070707074, + "loss": 1.4194, + "theoretical_loss": 3.425788805431309, + "tokens_seen": 2032533504 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906868686868687, + "loss": 1.327, + "theoretical_loss": 3.4257796249504375, + "tokens_seen": 2032599040 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039066666666666666, + "loss": 1.5645, + "theoretical_loss": 3.42577044484844, + "tokens_seen": 2032664576 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906464646464646, + "loss": 1.5047, + "theoretical_loss": 3.425761265125288, + "tokens_seen": 2032730112 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906262626262626, + "loss": 1.464, + "theoretical_loss": 3.4257520857809545, + "tokens_seen": 2032795648 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039060606060606064, + "loss": 1.3326, + "theoretical_loss": 3.4257429068154113, + "tokens_seen": 2032861184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003905858585858586, + "loss": 1.5455, + "theoretical_loss": 3.4257337282286304, + "tokens_seen": 2032926720 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039056565656565655, + "loss": 1.317, + "theoretical_loss": 3.4257245500205844, + "tokens_seen": 2032992256 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039054545454545456, + "loss": 1.5326, + "theoretical_loss": 3.4257153721912452, + "tokens_seen": 2033057792 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003905252525252525, + "loss": 1.5445, + "theoretical_loss": 3.4257061947405845, + "tokens_seen": 2033123328 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039050505050505053, + "loss": 1.182, + "theoretical_loss": 3.4256970176685755, + "tokens_seen": 2033188864 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904848484848485, + "loss": 1.4214, + "theoretical_loss": 3.42568784097519, + "tokens_seen": 2033254400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039046464646464644, + "loss": 1.3142, + "theoretical_loss": 3.425678664660399, + "tokens_seen": 2033319936 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039044444444444445, + "loss": 1.386, + "theoretical_loss": 3.4256694887241768, + "tokens_seen": 2033385472 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039042424242424247, + "loss": 1.426, + "theoretical_loss": 3.4256603131664933, + "tokens_seen": 2033451008 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.3975439965724945, + "objective/train/avg_token_score": 0.041283704340457916, + "objective/train/docs_used": 1149676, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4140899181365967, + "objective/train/original_loss": 4.860000133514404, + "objective/train/theoretical_loss": 3.4256511379873227, + "objective/train/tokens_used": 392375776, + "objective/train/unlikelihood_loss": 0.06886297464370728, + "theoretical_loss": 3.4256511379873227, + "tokens_seen": 2033516544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904040404040404, + "loss": 1.3443, + "theoretical_loss": 3.4256511379873227, + "tokens_seen": 2033516544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903838383838384, + "loss": 1.3114, + "theoretical_loss": 3.425641963186636, + "tokens_seen": 2033582080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903636363636364, + "loss": 1.5314, + "theoretical_loss": 3.425632788764406, + "tokens_seen": 2033647616 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039034343434343435, + "loss": 1.2772, + "theoretical_loss": 3.425623614720604, + "tokens_seen": 2033713152 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039032323232323236, + "loss": 1.3318, + "theoretical_loss": 3.4256144410552034, + "tokens_seen": 2033778688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903030303030303, + "loss": 1.412, + "theoretical_loss": 3.4256052677681756, + "tokens_seen": 2033844224 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039028282828282827, + "loss": 1.3957, + "theoretical_loss": 3.4255960948594932, + "tokens_seen": 2033909760 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902626262626263, + "loss": 1.5125, + "theoretical_loss": 3.425586922329128, + "tokens_seen": 2033975296 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039024242424242424, + "loss": 1.5286, + "theoretical_loss": 3.4255777501770526, + "tokens_seen": 2034040832 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039022222222222225, + "loss": 1.5622, + "theoretical_loss": 3.4255685784032393, + "tokens_seen": 2034106368 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902020202020202, + "loss": 1.3586, + "theoretical_loss": 3.4255594070076594, + "tokens_seen": 2034171904 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901818181818182, + "loss": 1.3422, + "theoretical_loss": 3.4255502359902863, + "tokens_seen": 2034237440 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901616161616162, + "loss": 1.3987, + "theoretical_loss": 3.4255410653510916, + "tokens_seen": 2034302976 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039014141414141413, + "loss": 1.2199, + "theoretical_loss": 3.425531895090047, + "tokens_seen": 2034368512 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039012121212121214, + "loss": 1.3739, + "theoretical_loss": 3.425522725207126, + "tokens_seen": 2034434048 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901010101010101, + "loss": 1.475, + "theoretical_loss": 3.4255135557022998, + "tokens_seen": 2034499584 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003900808080808081, + "loss": 1.256, + "theoretical_loss": 3.4255043865755415, + "tokens_seen": 2034565120 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039006060606060607, + "loss": 1.4305, + "theoretical_loss": 3.425495217826822, + "tokens_seen": 2034630656 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390040404040404, + "loss": 1.4553, + "theoretical_loss": 3.425486049456115, + "tokens_seen": 2034696192 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039002020202020204, + "loss": 1.5309, + "theoretical_loss": 3.425476881463392, + "tokens_seen": 2034761728 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039000000000000005, + "loss": 1.3208, + "theoretical_loss": 3.425467713848625, + "tokens_seen": 2034827264 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389979797979798, + "loss": 1.4335, + "theoretical_loss": 3.425458546611787, + "tokens_seen": 2034892800 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038995959595959596, + "loss": 1.6035, + "theoretical_loss": 3.42544937975285, + "tokens_seen": 2034958336 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899393939393939, + "loss": 1.303, + "theoretical_loss": 3.4254402132717856, + "tokens_seen": 2035023872 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038991919191919193, + "loss": 1.5508, + "theoretical_loss": 3.4254310471685665, + "tokens_seen": 2035089408 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.40835776925086975, + "objective/train/avg_token_score": 0.025650721043348312, + "objective/train/docs_used": 1150523, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4754818677902222, + "objective/train/original_loss": 4.815614223480225, + "objective/train/theoretical_loss": 3.4254218814431656, + "objective/train/tokens_used": 394014176, + "objective/train/unlikelihood_loss": 0.04576752707362175, + "theoretical_loss": 3.4254218814431656, + "tokens_seen": 2035154944 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038989898989898994, + "loss": 1.3807, + "theoretical_loss": 3.4254218814431656, + "tokens_seen": 2035154944 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898787878787879, + "loss": 1.6743, + "theoretical_loss": 3.425412716095554, + "tokens_seen": 2035220480 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038985858585858585, + "loss": 1.4136, + "theoretical_loss": 3.4254035511257053, + "tokens_seen": 2035286016 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898383838383838, + "loss": 1.4809, + "theoretical_loss": 3.4253943865335903, + "tokens_seen": 2035351552 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898181818181819, + "loss": 1.3785, + "theoretical_loss": 3.425385222319182, + "tokens_seen": 2035417088 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038979797979797983, + "loss": 1.5739, + "theoretical_loss": 3.425376058482453, + "tokens_seen": 2035482624 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897777777777778, + "loss": 1.1614, + "theoretical_loss": 3.425366895023375, + "tokens_seen": 2035548160 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038975757575757575, + "loss": 1.3595, + "theoretical_loss": 3.425357731941921, + "tokens_seen": 2035613696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897373737373737, + "loss": 1.4393, + "theoretical_loss": 3.425348569238062, + "tokens_seen": 2035679232 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038971717171717177, + "loss": 1.2865, + "theoretical_loss": 3.4253394069117715, + "tokens_seen": 2035744768 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003896969696969697, + "loss": 1.499, + "theoretical_loss": 3.425330244963021, + "tokens_seen": 2035810304 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003896767676767677, + "loss": 1.568, + "theoretical_loss": 3.4253210833917835, + "tokens_seen": 2035875840 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038965656565656564, + "loss": 1.4883, + "theoretical_loss": 3.425311922198031, + "tokens_seen": 2035941376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003896363636363636, + "loss": 1.4195, + "theoretical_loss": 3.4253027613817357, + "tokens_seen": 2036006912 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038961616161616166, + "loss": 1.4323, + "theoretical_loss": 3.4252936009428696, + "tokens_seen": 2036072448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003895959595959596, + "loss": 1.3823, + "theoretical_loss": 3.4252844408814056, + "tokens_seen": 2036137984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003895757575757576, + "loss": 1.3188, + "theoretical_loss": 3.4252752811973157, + "tokens_seen": 2036203520 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038955555555555553, + "loss": 1.4645, + "theoretical_loss": 3.4252661218905724, + "tokens_seen": 2036269056 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038953535353535354, + "loss": 1.3862, + "theoretical_loss": 3.4252569629611473, + "tokens_seen": 2036334592 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038951515151515155, + "loss": 1.5763, + "theoretical_loss": 3.4252478044090138, + "tokens_seen": 2036400128 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894949494949495, + "loss": 1.1362, + "theoretical_loss": 3.4252386462341438, + "tokens_seen": 2036465664 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038947474747474747, + "loss": 1.5676, + "theoretical_loss": 3.4252294884365093, + "tokens_seen": 2036531200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894545454545454, + "loss": 1.6026, + "theoretical_loss": 3.425220331016083, + "tokens_seen": 2036596736 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038943434343434344, + "loss": 1.3879, + "theoretical_loss": 3.4252111739728366, + "tokens_seen": 2036662272 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038941414141414145, + "loss": 1.5851, + "theoretical_loss": 3.4252020173067432, + "tokens_seen": 2036727808 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.3683406710624695, + "objective/train/avg_token_score": 0.011941329576075077, + "objective/train/docs_used": 1151732, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1528146266937256, + "objective/train/original_loss": 4.464666843414307, + "objective/train/theoretical_loss": 3.4251928610177744, + "objective/train/tokens_used": 395652576, + "objective/train/unlikelihood_loss": 0.0723705142736435, + "theoretical_loss": 3.4251928610177744, + "tokens_seen": 2036793344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893939393939394, + "loss": 1.3575, + "theoretical_loss": 3.4251928610177744, + "tokens_seen": 2036793344 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038937373737373736, + "loss": 1.1522, + "theoretical_loss": 3.4251837051059035, + "tokens_seen": 2036858880 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038935353535353537, + "loss": 1.2316, + "theoretical_loss": 3.425174549571102, + "tokens_seen": 2036924416 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038933333333333333, + "loss": 1.2988, + "theoretical_loss": 3.4251653944133427, + "tokens_seen": 2036989952 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038931313131313134, + "loss": 1.4529, + "theoretical_loss": 3.4251562396325976, + "tokens_seen": 2037055488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892929292929293, + "loss": 1.6243, + "theoretical_loss": 3.4251470852288395, + "tokens_seen": 2037121024 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038927272727272725, + "loss": 1.3962, + "theoretical_loss": 3.4251379312020402, + "tokens_seen": 2037186560 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038925252525252526, + "loss": 1.6076, + "theoretical_loss": 3.4251287775521724, + "tokens_seen": 2037252096 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892323232323232, + "loss": 1.4508, + "theoretical_loss": 3.4251196242792084, + "tokens_seen": 2037317632 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038921212121212123, + "loss": 1.3475, + "theoretical_loss": 3.42511047138312, + "tokens_seen": 2037383168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891919191919192, + "loss": 1.4607, + "theoretical_loss": 3.425101318863881, + "tokens_seen": 2037448704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891717171717172, + "loss": 1.4235, + "theoretical_loss": 3.4250921667214627, + "tokens_seen": 2037514240 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038915151515151516, + "loss": 1.3172, + "theoretical_loss": 3.4250830149558373, + "tokens_seen": 2037579776 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891313131313131, + "loss": 1.2515, + "theoretical_loss": 3.4250738635669773, + "tokens_seen": 2037645312 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891111111111111, + "loss": 1.2808, + "theoretical_loss": 3.425064712554856, + "tokens_seen": 2037710848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890909090909091, + "loss": 1.0805, + "theoretical_loss": 3.425055561919444, + "tokens_seen": 2037776384 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890707070707071, + "loss": 1.4789, + "theoretical_loss": 3.4250464116607153, + "tokens_seen": 2037841920 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038905050505050505, + "loss": 1.2794, + "theoretical_loss": 3.4250372617786415, + "tokens_seen": 2037907456 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389030303030303, + "loss": 1.4194, + "theoretical_loss": 3.4250281122731954, + "tokens_seen": 2037972992 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389010101010101, + "loss": 1.433, + "theoretical_loss": 3.4250189631443493, + "tokens_seen": 2038038528 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038898989898989903, + "loss": 1.4705, + "theoretical_loss": 3.425009814392075, + "tokens_seen": 2038104064 + }, + { + "epoch": 0.23, + "learning_rate": 0.000388969696969697, + "loss": 1.3959, + "theoretical_loss": 3.425000666016346, + "tokens_seen": 2038169600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038894949494949494, + "loss": 1.4009, + "theoretical_loss": 3.4249915180171335, + "tokens_seen": 2038235136 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889292929292929, + "loss": 1.2109, + "theoretical_loss": 3.4249823703944102, + "tokens_seen": 2038300672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889090909090909, + "loss": 1.6845, + "theoretical_loss": 3.4249732231481493, + "tokens_seen": 2038366208 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.2941104471683502, + "objective/train/avg_token_score": 0.020645203068852425, + "objective/train/docs_used": 1152471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9252520203590393, + "objective/train/original_loss": 4.827944278717041, + "objective/train/theoretical_loss": 3.4249640762783224, + "objective/train/tokens_used": 397290976, + "objective/train/unlikelihood_loss": 0.03434165567159653, + "theoretical_loss": 3.4249640762783224, + "tokens_seen": 2038431744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888888888888889, + "loss": 1.2632, + "theoretical_loss": 3.4249640762783224, + "tokens_seen": 2038431744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888686868686869, + "loss": 1.6081, + "theoretical_loss": 3.424954929784902, + "tokens_seen": 2038497280 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038884848484848483, + "loss": 1.3837, + "theoretical_loss": 3.424945783667861, + "tokens_seen": 2038562816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888282828282828, + "loss": 1.5471, + "theoretical_loss": 3.4249366379271713, + "tokens_seen": 2038628352 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038880808080808086, + "loss": 1.455, + "theoretical_loss": 3.4249274925628055, + "tokens_seen": 2038693888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887878787878788, + "loss": 1.5115, + "theoretical_loss": 3.424918347574736, + "tokens_seen": 2038759424 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038876767676767677, + "loss": 1.6937, + "theoretical_loss": 3.4249092029629353, + "tokens_seen": 2038824960 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038874747474747473, + "loss": 1.3687, + "theoretical_loss": 3.424900058727376, + "tokens_seen": 2038890496 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038872727272727274, + "loss": 1.7146, + "theoretical_loss": 3.4248909148680298, + "tokens_seen": 2038956032 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038870707070707075, + "loss": 1.5177, + "theoretical_loss": 3.42488177138487, + "tokens_seen": 2039021568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886868686868687, + "loss": 1.4942, + "theoretical_loss": 3.424872628277868, + "tokens_seen": 2039087104 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038866666666666666, + "loss": 1.5334, + "theoretical_loss": 3.4248634855469975, + "tokens_seen": 2039152640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886464646464646, + "loss": 1.3483, + "theoretical_loss": 3.42485434319223, + "tokens_seen": 2039218176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886262626262627, + "loss": 1.3849, + "theoretical_loss": 3.4248452012135386, + "tokens_seen": 2039283712 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038860606060606064, + "loss": 1.4504, + "theoretical_loss": 3.424836059610895, + "tokens_seen": 2039349248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885858585858586, + "loss": 1.4475, + "theoretical_loss": 3.424826918384272, + "tokens_seen": 2039414784 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038856565656565656, + "loss": 1.5788, + "theoretical_loss": 3.424817777533643, + "tokens_seen": 2039480320 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885454545454545, + "loss": 1.3884, + "theoretical_loss": 3.4248086370589785, + "tokens_seen": 2039545856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885252525252526, + "loss": 1.6731, + "theoretical_loss": 3.4247994969602527, + "tokens_seen": 2039611392 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038850505050505054, + "loss": 1.3777, + "theoretical_loss": 3.4247903572374367, + "tokens_seen": 2039676928 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884848484848485, + "loss": 1.5438, + "theoretical_loss": 3.424781217890504, + "tokens_seen": 2039742464 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038846464646464645, + "loss": 1.052, + "theoretical_loss": 3.4247720789194274, + "tokens_seen": 2039808000 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038844444444444446, + "loss": 1.3154, + "theoretical_loss": 3.424762940324178, + "tokens_seen": 2039873536 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038842424242424247, + "loss": 1.2718, + "theoretical_loss": 3.424753802104729, + "tokens_seen": 2039939072 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038840404040404043, + "loss": 1.4838, + "theoretical_loss": 3.4247446642610524, + "tokens_seen": 2040004608 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.39326736330986023, + "objective/train/avg_token_score": 0.01161976158618927, + "objective/train/docs_used": 1153876, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2003021240234375, + "objective/train/original_loss": 4.390503883361816, + "objective/train/theoretical_loss": 3.4247355267931217, + "objective/train/tokens_used": 398929376, + "objective/train/unlikelihood_loss": 0.05784899368882179, + "theoretical_loss": 3.4247355267931217, + "tokens_seen": 2040070144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883838383838384, + "loss": 1.3673, + "theoretical_loss": 3.4247355267931217, + "tokens_seen": 2040070144 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038836363636363634, + "loss": 1.2824, + "theoretical_loss": 3.4247263897009086, + "tokens_seen": 2040135680 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038834343434343435, + "loss": 1.4723, + "theoretical_loss": 3.4247172529843857, + "tokens_seen": 2040201216 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038832323232323236, + "loss": 1.3064, + "theoretical_loss": 3.4247081166435254, + "tokens_seen": 2040266752 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883030303030303, + "loss": 1.5967, + "theoretical_loss": 3.4246989806783006, + "tokens_seen": 2040332288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882828282828283, + "loss": 1.4823, + "theoretical_loss": 3.4246898450886833, + "tokens_seen": 2040397824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882626262626263, + "loss": 1.1834, + "theoretical_loss": 3.4246807098746466, + "tokens_seen": 2040463360 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038824242424242425, + "loss": 1.4359, + "theoretical_loss": 3.424671575036162, + "tokens_seen": 2040528896 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038822222222222226, + "loss": 1.3799, + "theoretical_loss": 3.424662440573203, + "tokens_seen": 2040594432 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882020202020202, + "loss": 1.1812, + "theoretical_loss": 3.4246533064857414, + "tokens_seen": 2040659968 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038818181818181817, + "loss": 1.6693, + "theoretical_loss": 3.4246441727737507, + "tokens_seen": 2040725504 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881616161616162, + "loss": 1.5425, + "theoretical_loss": 3.424635039437202, + "tokens_seen": 2040791040 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038814141414141414, + "loss": 1.3501, + "theoretical_loss": 3.4246259064760687, + "tokens_seen": 2040856576 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038812121212121215, + "loss": 1.4213, + "theoretical_loss": 3.4246167738903233, + "tokens_seen": 2040922112 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881010101010101, + "loss": 1.2594, + "theoretical_loss": 3.424607641679938, + "tokens_seen": 2040987648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003880808080808081, + "loss": 1.407, + "theoretical_loss": 3.424598509844886, + "tokens_seen": 2041053184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003880606060606061, + "loss": 1.5835, + "theoretical_loss": 3.4245893783851384, + "tokens_seen": 2041118720 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038804040404040403, + "loss": 1.484, + "theoretical_loss": 3.4245802473006695, + "tokens_seen": 2041184256 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038802020202020204, + "loss": 1.5434, + "theoretical_loss": 3.4245711165914505, + "tokens_seen": 2041249792 + }, + { + "epoch": 0.23, + "learning_rate": 0.000388, + "loss": 1.5105, + "theoretical_loss": 3.4245619862574546, + "tokens_seen": 2041315328 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387979797979798, + "loss": 1.4721, + "theoretical_loss": 3.424552856298654, + "tokens_seen": 2041380864 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038795959595959597, + "loss": 1.338, + "theoretical_loss": 3.4245437267150214, + "tokens_seen": 2041446400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879393939393939, + "loss": 1.5508, + "theoretical_loss": 3.4245345975065296, + "tokens_seen": 2041511936 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038791919191919193, + "loss": 1.3857, + "theoretical_loss": 3.4245254686731506, + "tokens_seen": 2041577472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878989898989899, + "loss": 1.6434, + "theoretical_loss": 3.424516340214857, + "tokens_seen": 2041643008 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.4324902296066284, + "objective/train/avg_token_score": 0.014002914540469646, + "objective/train/docs_used": 1154501, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5224765539169312, + "objective/train/original_loss": 4.884289264678955, + "objective/train/theoretical_loss": 3.424507212131622, + "objective/train/tokens_used": 400567776, + "objective/train/unlikelihood_loss": 0.04442446678876877, + "theoretical_loss": 3.424507212131622, + "tokens_seen": 2041708544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878787878787879, + "loss": 1.4539, + "theoretical_loss": 3.424507212131622, + "tokens_seen": 2041708544 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038785858585858586, + "loss": 1.3583, + "theoretical_loss": 3.4244980844234174, + "tokens_seen": 2041774080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878383838383838, + "loss": 1.3776, + "theoretical_loss": 3.424488957090216, + "tokens_seen": 2041839616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878181818181818, + "loss": 1.4747, + "theoretical_loss": 3.4244798301319905, + "tokens_seen": 2041905152 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038779797979797984, + "loss": 1.2291, + "theoretical_loss": 3.424470703548714, + "tokens_seen": 2041970688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877777777777778, + "loss": 1.2546, + "theoretical_loss": 3.4244615773403577, + "tokens_seen": 2042036224 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038775757575757575, + "loss": 1.2885, + "theoretical_loss": 3.4244524515068946, + "tokens_seen": 2042101760 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877373737373737, + "loss": 1.3927, + "theoretical_loss": 3.4244433260482983, + "tokens_seen": 2042167296 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877171717171717, + "loss": 1.5276, + "theoretical_loss": 3.4244342009645403, + "tokens_seen": 2042232832 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038769696969696973, + "loss": 1.3749, + "theoretical_loss": 3.4244250762555937, + "tokens_seen": 2042298368 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876767676767677, + "loss": 1.3225, + "theoretical_loss": 3.424415951921431, + "tokens_seen": 2042363904 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038765656565656564, + "loss": 1.3656, + "theoretical_loss": 3.4244068279620246, + "tokens_seen": 2042429440 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876363636363636, + "loss": 1.4351, + "theoretical_loss": 3.4243977043773466, + "tokens_seen": 2042494976 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038761616161616167, + "loss": 1.51, + "theoretical_loss": 3.4243885811673707, + "tokens_seen": 2042560512 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875959595959596, + "loss": 1.5127, + "theoretical_loss": 3.424379458332069, + "tokens_seen": 2042626048 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875757575757576, + "loss": 1.4925, + "theoretical_loss": 3.424370335871414, + "tokens_seen": 2042691584 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038755555555555554, + "loss": 1.6616, + "theoretical_loss": 3.424361213785378, + "tokens_seen": 2042757120 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875353535353535, + "loss": 1.4192, + "theoretical_loss": 3.424352092073934, + "tokens_seen": 2042822656 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038751515151515156, + "loss": 1.5003, + "theoretical_loss": 3.424342970737055, + "tokens_seen": 2042888192 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874949494949495, + "loss": 1.5405, + "theoretical_loss": 3.4243338497747127, + "tokens_seen": 2042953728 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874747474747475, + "loss": 1.5091, + "theoretical_loss": 3.42432472918688, + "tokens_seen": 2043019264 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038745454545454543, + "loss": 1.5408, + "theoretical_loss": 3.4243156089735303, + "tokens_seen": 2043084800 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038743434343434344, + "loss": 1.5459, + "theoretical_loss": 3.424306489134635, + "tokens_seen": 2043150336 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038741414141414145, + "loss": 1.3214, + "theoretical_loss": 3.4242973696701675, + "tokens_seen": 2043215872 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873939393939394, + "loss": 1.3561, + "theoretical_loss": 3.4242882505801, + "tokens_seen": 2043281408 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.3992546498775482, + "objective/train/avg_token_score": 0.018309487029910088, + "objective/train/docs_used": 1155127, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4903119802474976, + "objective/train/original_loss": 4.612830638885498, + "objective/train/theoretical_loss": 3.424279131864406, + "objective/train/tokens_used": 402206176, + "objective/train/unlikelihood_loss": 0.05610045790672302, + "theoretical_loss": 3.424279131864406, + "tokens_seen": 2043346944 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038737373737373737, + "loss": 1.5274, + "theoretical_loss": 3.424279131864406, + "tokens_seen": 2043346944 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873535353535353, + "loss": 1.4273, + "theoretical_loss": 3.4242700135230564, + "tokens_seen": 2043412480 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038733333333333333, + "loss": 1.3918, + "theoretical_loss": 3.424260895556025, + "tokens_seen": 2043478016 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038731313131313134, + "loss": 1.3255, + "theoretical_loss": 3.424251777963285, + "tokens_seen": 2043543552 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872929292929293, + "loss": 1.4575, + "theoretical_loss": 3.4242426607448078, + "tokens_seen": 2043609088 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038727272727272726, + "loss": 1.4562, + "theoretical_loss": 3.424233543900567, + "tokens_seen": 2043674624 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038725252525252527, + "loss": 1.4065, + "theoretical_loss": 3.4242244274305347, + "tokens_seen": 2043740160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872323232323232, + "loss": 1.3534, + "theoretical_loss": 3.4242153113346836, + "tokens_seen": 2043805696 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038721212121212124, + "loss": 1.382, + "theoretical_loss": 3.424206195612986, + "tokens_seen": 2043871232 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871919191919192, + "loss": 1.176, + "theoretical_loss": 3.424197080265415, + "tokens_seen": 2043936768 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038717171717171715, + "loss": 1.4214, + "theoretical_loss": 3.424187965291944, + "tokens_seen": 2044002304 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038715151515151516, + "loss": 1.4899, + "theoretical_loss": 3.4241788506925444, + "tokens_seen": 2044067840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871313131313131, + "loss": 1.5427, + "theoretical_loss": 3.4241697364671886, + "tokens_seen": 2044133376 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038711111111111113, + "loss": 1.4991, + "theoretical_loss": 3.424160622615851, + "tokens_seen": 2044198912 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870909090909091, + "loss": 1.3178, + "theoretical_loss": 3.424151509138502, + "tokens_seen": 2044264448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870707070707071, + "loss": 1.4449, + "theoretical_loss": 3.4241423960351165, + "tokens_seen": 2044329984 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038705050505050505, + "loss": 1.1632, + "theoretical_loss": 3.4241332833056664, + "tokens_seen": 2044395520 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038703030303030307, + "loss": 1.251, + "theoretical_loss": 3.4241241709501233, + "tokens_seen": 2044461056 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387010101010101, + "loss": 1.3892, + "theoretical_loss": 3.424115058968461, + "tokens_seen": 2044526592 + }, + { + "epoch": 0.23, + "learning_rate": 0.000386989898989899, + "loss": 1.2911, + "theoretical_loss": 3.424105947360652, + "tokens_seen": 2044592128 + }, + { + "epoch": 0.23, + "learning_rate": 0.000386969696969697, + "loss": 1.5039, + "theoretical_loss": 3.424096836126669, + "tokens_seen": 2044657664 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038694949494949495, + "loss": 1.4078, + "theoretical_loss": 3.4240877252664834, + "tokens_seen": 2044723200 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038692929292929296, + "loss": 1.5386, + "theoretical_loss": 3.42407861478007, + "tokens_seen": 2044788736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869090909090909, + "loss": 1.6039, + "theoretical_loss": 3.4240695046674006, + "tokens_seen": 2044854272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003868888888888889, + "loss": 1.4234, + "theoretical_loss": 3.424060394928447, + "tokens_seen": 2044919808 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.4493523836135864, + "objective/train/avg_token_score": 0.012142237275838852, + "objective/train/docs_used": 1156711, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4567104578018188, + "objective/train/original_loss": 3.7896244525909424, + "objective/train/theoretical_loss": 3.4240512855631833, + "objective/train/tokens_used": 403844576, + "objective/train/unlikelihood_loss": 0.09484424442052841, + "theoretical_loss": 3.4240512855631833, + "tokens_seen": 2044985344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003868686868686869, + "loss": 1.4923, + "theoretical_loss": 3.4240512855631833, + "tokens_seen": 2044985344 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038684848484848484, + "loss": 1.4911, + "theoretical_loss": 3.424042176571582, + "tokens_seen": 2045050880 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038682828282828285, + "loss": 1.5525, + "theoretical_loss": 3.424033067953615, + "tokens_seen": 2045116416 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003868080808080808, + "loss": 1.598, + "theoretical_loss": 3.4240239597092548, + "tokens_seen": 2045181952 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867878787878788, + "loss": 1.5819, + "theoretical_loss": 3.424014851838475, + "tokens_seen": 2045247488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867676767676768, + "loss": 1.427, + "theoretical_loss": 3.4240057443412484, + "tokens_seen": 2045313024 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038674747474747473, + "loss": 1.41, + "theoretical_loss": 3.423996637217547, + "tokens_seen": 2045378560 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038672727272727274, + "loss": 1.4549, + "theoretical_loss": 3.4239875304673437, + "tokens_seen": 2045444096 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038670707070707076, + "loss": 1.4496, + "theoretical_loss": 3.4239784240906115, + "tokens_seen": 2045509632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003866868686868687, + "loss": 1.304, + "theoretical_loss": 3.423969318087323, + "tokens_seen": 2045575168 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038666666666666667, + "loss": 1.2372, + "theoretical_loss": 3.423960212457451, + "tokens_seen": 2045640704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003866464646464646, + "loss": 1.5265, + "theoretical_loss": 3.423951107200968, + "tokens_seen": 2045706240 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038662626262626264, + "loss": 1.1508, + "theoretical_loss": 3.423942002317846, + "tokens_seen": 2045771776 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038660606060606065, + "loss": 1.5112, + "theoretical_loss": 3.4239328978080596, + "tokens_seen": 2045837312 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865858585858586, + "loss": 1.3431, + "theoretical_loss": 3.4239237936715803, + "tokens_seen": 2045902848 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038656565656565656, + "loss": 1.434, + "theoretical_loss": 3.4239146899083805, + "tokens_seen": 2045968384 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865454545454545, + "loss": 1.4307, + "theoretical_loss": 3.4239055865184342, + "tokens_seen": 2046033920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865252525252526, + "loss": 1.2533, + "theoretical_loss": 3.423896483501713, + "tokens_seen": 2046099456 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038650505050505054, + "loss": 1.2292, + "theoretical_loss": 3.42388738085819, + "tokens_seen": 2046164992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864848484848485, + "loss": 1.329, + "theoretical_loss": 3.423878278587838, + "tokens_seen": 2046230528 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038646464646464645, + "loss": 1.4768, + "theoretical_loss": 3.42386917669063, + "tokens_seen": 2046296064 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864444444444444, + "loss": 1.4991, + "theoretical_loss": 3.4238600751665382, + "tokens_seen": 2046361600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864242424242425, + "loss": 1.2756, + "theoretical_loss": 3.4238509740155356, + "tokens_seen": 2046427136 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038640404040404043, + "loss": 1.3897, + "theoretical_loss": 3.4238418732375955, + "tokens_seen": 2046492672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003863838383838384, + "loss": 1.475, + "theoretical_loss": 3.42383277283269, + "tokens_seen": 2046558208 + }, + { + "epoch": 0.23, + "objective/train/aligned_token_count": 0.402309387922287, + "objective/train/avg_token_score": 0.005781781394034624, + "objective/train/docs_used": 1157531, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2796791791915894, + "objective/train/original_loss": 3.6973581314086914, + "objective/train/theoretical_loss": 3.4238236728007916, + "objective/train/tokens_used": 405482976, + "objective/train/unlikelihood_loss": 0.09952288120985031, + "theoretical_loss": 3.4238236728007916, + "tokens_seen": 2046623744 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038636363636363635, + "loss": 1.3517, + "theoretical_loss": 3.4238236728007916, + "tokens_seen": 2046623744 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003863434343434343, + "loss": 1.473, + "theoretical_loss": 3.4238145731418737, + "tokens_seen": 2046689280 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038632323232323237, + "loss": 1.2185, + "theoretical_loss": 3.4238054738559094, + "tokens_seen": 2046754816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003863030303030303, + "loss": 1.4437, + "theoretical_loss": 3.4237963749428704, + "tokens_seen": 2046820352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862828282828283, + "loss": 1.5039, + "theoretical_loss": 3.4237872764027304, + "tokens_seen": 2046885888 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038626262626262624, + "loss": 1.4342, + "theoretical_loss": 3.4237781782354615, + "tokens_seen": 2046951424 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038624242424242425, + "loss": 1.4443, + "theoretical_loss": 3.423769080441037, + "tokens_seen": 2047016960 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038622222222222226, + "loss": 1.4982, + "theoretical_loss": 3.423759983019429, + "tokens_seen": 2047082496 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862020202020202, + "loss": 1.4878, + "theoretical_loss": 3.4237508859706116, + "tokens_seen": 2047148032 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861818181818182, + "loss": 1.4705, + "theoretical_loss": 3.423741789294556, + "tokens_seen": 2047213568 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038616161616161613, + "loss": 1.358, + "theoretical_loss": 3.423732692991236, + "tokens_seen": 2047279104 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038614141414141414, + "loss": 1.5082, + "theoretical_loss": 3.4237235970606243, + "tokens_seen": 2047344640 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038612121212121215, + "loss": 1.3321, + "theoretical_loss": 3.4237145015026935, + "tokens_seen": 2047410176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861010101010101, + "loss": 1.4646, + "theoretical_loss": 3.4237054063174166, + "tokens_seen": 2047475712 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038608080808080807, + "loss": 1.5084, + "theoretical_loss": 3.4236963115047656, + "tokens_seen": 2047541248 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860606060606061, + "loss": 1.5789, + "theoretical_loss": 3.423687217064715, + "tokens_seen": 2047606784 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038604040404040404, + "loss": 1.4218, + "theoretical_loss": 3.4236781229972353, + "tokens_seen": 2047672320 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038602020202020205, + "loss": 1.2997, + "theoretical_loss": 3.4236690293023018, + "tokens_seen": 2047737856 + }, + { + "epoch": 0.24, + "learning_rate": 0.000386, + "loss": 1.3044, + "theoretical_loss": 3.4236599359798854, + "tokens_seen": 2047803392 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038597979797979796, + "loss": 1.3948, + "theoretical_loss": 3.42365084302996, + "tokens_seen": 2047868928 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038595959595959597, + "loss": 1.5736, + "theoretical_loss": 3.4236417504524974, + "tokens_seen": 2047934464 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038593939393939393, + "loss": 1.5154, + "theoretical_loss": 3.4236326582474716, + "tokens_seen": 2048000000 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038591919191919194, + "loss": 1.1755, + "theoretical_loss": 3.4236235664148547, + "tokens_seen": 2048065536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858989898989899, + "loss": 1.4525, + "theoretical_loss": 3.42361447495462, + "tokens_seen": 2048131072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858787878787879, + "loss": 1.2833, + "theoretical_loss": 3.4236053838667395, + "tokens_seen": 2048196608 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.46884164214134216, + "objective/train/avg_token_score": 0.013134431093931198, + "objective/train/docs_used": 1159006, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5853708982467651, + "objective/train/original_loss": 4.141778469085693, + "objective/train/theoretical_loss": 3.4235962931511867, + "objective/train/tokens_used": 407121376, + "objective/train/unlikelihood_loss": 0.04517946392297745, + "theoretical_loss": 3.4235962931511867, + "tokens_seen": 2048262144 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038585858585858586, + "loss": 1.5279, + "theoretical_loss": 3.4235962931511867, + "tokens_seen": 2048262144 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858383838383838, + "loss": 1.3556, + "theoretical_loss": 3.4235872028079353, + "tokens_seen": 2048327680 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038581818181818183, + "loss": 1.5147, + "theoretical_loss": 3.423578112836956, + "tokens_seen": 2048393216 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857979797979798, + "loss": 1.3604, + "theoretical_loss": 3.4235690232382234, + "tokens_seen": 2048458752 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857777777777778, + "loss": 1.6133, + "theoretical_loss": 3.4235599340117098, + "tokens_seen": 2048524288 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038575757575757576, + "loss": 1.2753, + "theoretical_loss": 3.4235508451573873, + "tokens_seen": 2048589824 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857373737373737, + "loss": 1.4364, + "theoretical_loss": 3.42354175667523, + "tokens_seen": 2048655360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857171717171717, + "loss": 1.4124, + "theoretical_loss": 3.42353266856521, + "tokens_seen": 2048720896 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038569696969696974, + "loss": 1.4019, + "theoretical_loss": 3.4235235808273012, + "tokens_seen": 2048786432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856767676767677, + "loss": 1.3403, + "theoretical_loss": 3.4235144934614747, + "tokens_seen": 2048851968 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038565656565656565, + "loss": 1.2462, + "theoretical_loss": 3.423505406467705, + "tokens_seen": 2048917504 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856363636363636, + "loss": 1.3504, + "theoretical_loss": 3.4234963198459636, + "tokens_seen": 2048983040 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856161616161616, + "loss": 1.4364, + "theoretical_loss": 3.4234872335962248, + "tokens_seen": 2049048576 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038559595959595963, + "loss": 1.4967, + "theoretical_loss": 3.42347814771846, + "tokens_seen": 2049114112 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855757575757576, + "loss": 1.3122, + "theoretical_loss": 3.423469062212643, + "tokens_seen": 2049179648 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038555555555555554, + "loss": 1.4711, + "theoretical_loss": 3.4234599770787466, + "tokens_seen": 2049245184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855353535353535, + "loss": 1.2197, + "theoretical_loss": 3.4234508923167435, + "tokens_seen": 2049310720 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038551515151515156, + "loss": 1.3143, + "theoretical_loss": 3.423441807926607, + "tokens_seen": 2049376256 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854949494949495, + "loss": 1.1005, + "theoretical_loss": 3.423432723908309, + "tokens_seen": 2049441792 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854747474747475, + "loss": 1.5098, + "theoretical_loss": 3.423423640261823, + "tokens_seen": 2049507328 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038545454545454544, + "loss": 1.4713, + "theoretical_loss": 3.4234145569871224, + "tokens_seen": 2049572864 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854343434343434, + "loss": 1.3695, + "theoretical_loss": 3.4234054740841797, + "tokens_seen": 2049638400 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038541414141414146, + "loss": 1.2553, + "theoretical_loss": 3.4233963915529673, + "tokens_seen": 2049703936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003853939393939394, + "loss": 1.6962, + "theoretical_loss": 3.4233873093934584, + "tokens_seen": 2049769472 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038537373737373737, + "loss": 1.5217, + "theoretical_loss": 3.4233782276056264, + "tokens_seen": 2049835008 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.45326244831085205, + "objective/train/avg_token_score": 0.009649938903748989, + "objective/train/docs_used": 1159734, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5877481698989868, + "objective/train/original_loss": 4.093586444854736, + "objective/train/theoretical_loss": 3.4233691461894438, + "objective/train/tokens_used": 408759776, + "objective/train/unlikelihood_loss": 0.0731666311621666, + "theoretical_loss": 3.4233691461894438, + "tokens_seen": 2049900544 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038535353535353533, + "loss": 1.4922, + "theoretical_loss": 3.4233691461894438, + "tokens_seen": 2049900544 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003853333333333334, + "loss": 1.2207, + "theoretical_loss": 3.4233600651448834, + "tokens_seen": 2049966080 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038531313131313135, + "loss": 1.1984, + "theoretical_loss": 3.4233509844719183, + "tokens_seen": 2050031616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852929292929293, + "loss": 1.3338, + "theoretical_loss": 3.4233419041705213, + "tokens_seen": 2050097152 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038527272727272726, + "loss": 1.3954, + "theoretical_loss": 3.4233328242406653, + "tokens_seen": 2050162688 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852525252525252, + "loss": 1.3318, + "theoretical_loss": 3.4233237446823237, + "tokens_seen": 2050228224 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852323232323233, + "loss": 1.2815, + "theoretical_loss": 3.4233146654954685, + "tokens_seen": 2050293760 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038521212121212124, + "loss": 1.4702, + "theoretical_loss": 3.4233055866800735, + "tokens_seen": 2050359296 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851919191919192, + "loss": 1.3674, + "theoretical_loss": 3.4232965082361106, + "tokens_seen": 2050424832 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038517171717171716, + "loss": 1.2718, + "theoretical_loss": 3.4232874301635543, + "tokens_seen": 2050490368 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038515151515151517, + "loss": 1.579, + "theoretical_loss": 3.4232783524623764, + "tokens_seen": 2050555904 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851313131313132, + "loss": 1.3102, + "theoretical_loss": 3.42326927513255, + "tokens_seen": 2050621440 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038511111111111114, + "loss": 1.4095, + "theoretical_loss": 3.423260198174048, + "tokens_seen": 2050686976 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850909090909091, + "loss": 1.6638, + "theoretical_loss": 3.4232511215868433, + "tokens_seen": 2050752512 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038507070707070705, + "loss": 1.3702, + "theoretical_loss": 3.42324204537091, + "tokens_seen": 2050818048 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038505050505050506, + "loss": 1.2757, + "theoretical_loss": 3.423232969526219, + "tokens_seen": 2050883584 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038503030303030307, + "loss": 1.1061, + "theoretical_loss": 3.423223894052745, + "tokens_seen": 2050949120 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038501010101010103, + "loss": 1.4858, + "theoretical_loss": 3.4232148189504596, + "tokens_seen": 2051014656 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384989898989899, + "loss": 1.2475, + "theoretical_loss": 3.423205744219337, + "tokens_seen": 2051080192 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384969696969697, + "loss": 1.3413, + "theoretical_loss": 3.4231966698593492, + "tokens_seen": 2051145728 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038494949494949495, + "loss": 1.525, + "theoretical_loss": 3.42318759587047, + "tokens_seen": 2051211264 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038492929292929296, + "loss": 1.4001, + "theoretical_loss": 3.423178522252672, + "tokens_seen": 2051276800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003849090909090909, + "loss": 1.5063, + "theoretical_loss": 3.4231694490059272, + "tokens_seen": 2051342336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848888888888889, + "loss": 1.1284, + "theoretical_loss": 3.4231603761302107, + "tokens_seen": 2051407872 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848686868686869, + "loss": 1.2998, + "theoretical_loss": 3.4231513036254935, + "tokens_seen": 2051473408 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.4117790758609772, + "objective/train/avg_token_score": 0.015326161868870258, + "objective/train/docs_used": 1160859, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1978123188018799, + "objective/train/original_loss": 3.7221078872680664, + "objective/train/theoretical_loss": 3.4231422314917497, + "objective/train/tokens_used": 410398176, + "objective/train/unlikelihood_loss": 0.08059471845626831, + "theoretical_loss": 3.4231422314917497, + "tokens_seen": 2051538944 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038484848484848485, + "loss": 1.3475, + "theoretical_loss": 3.4231422314917497, + "tokens_seen": 2051538944 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038482828282828286, + "loss": 1.3863, + "theoretical_loss": 3.423133159728951, + "tokens_seen": 2051604480 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848080808080808, + "loss": 1.4549, + "theoretical_loss": 3.4231240883370724, + "tokens_seen": 2051670016 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038478787878787877, + "loss": 1.1667, + "theoretical_loss": 3.423115017316085, + "tokens_seen": 2051735552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847676767676768, + "loss": 1.3304, + "theoretical_loss": 3.423105946665963, + "tokens_seen": 2051801088 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038474747474747474, + "loss": 1.4697, + "theoretical_loss": 3.423096876386679, + "tokens_seen": 2051866624 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038472727272727275, + "loss": 1.3162, + "theoretical_loss": 3.4230878064782058, + "tokens_seen": 2051932160 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847070707070707, + "loss": 1.4495, + "theoretical_loss": 3.4230787369405165, + "tokens_seen": 2051997696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846868686868687, + "loss": 1.4223, + "theoretical_loss": 3.4230696677735843, + "tokens_seen": 2052063232 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846666666666667, + "loss": 1.4094, + "theoretical_loss": 3.423060598977382, + "tokens_seen": 2052128768 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038464646464646463, + "loss": 1.7047, + "theoretical_loss": 3.4230515305518825, + "tokens_seen": 2052194304 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038462626262626264, + "loss": 1.4611, + "theoretical_loss": 3.423042462497059, + "tokens_seen": 2052259840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846060606060606, + "loss": 1.3121, + "theoretical_loss": 3.423033394812885, + "tokens_seen": 2052325376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845858585858586, + "loss": 1.2598, + "theoretical_loss": 3.4230243274993324, + "tokens_seen": 2052390912 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038456565656565657, + "loss": 1.3826, + "theoretical_loss": 3.423015260556375, + "tokens_seen": 2052456448 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845454545454545, + "loss": 1.3184, + "theoretical_loss": 3.4230061939839858, + "tokens_seen": 2052521984 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038452525252525254, + "loss": 1.37, + "theoretical_loss": 3.4229971277821374, + "tokens_seen": 2052587520 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038450505050505055, + "loss": 1.4403, + "theoretical_loss": 3.4229880619508033, + "tokens_seen": 2052653056 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844848484848485, + "loss": 1.4237, + "theoretical_loss": 3.422978996489956, + "tokens_seen": 2052718592 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038446464646464646, + "loss": 1.4622, + "theoretical_loss": 3.422969931399569, + "tokens_seen": 2052784128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844444444444444, + "loss": 1.273, + "theoretical_loss": 3.422960866679616, + "tokens_seen": 2052849664 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038442424242424243, + "loss": 1.5379, + "theoretical_loss": 3.422951802330068, + "tokens_seen": 2052915200 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038440404040404044, + "loss": 1.2839, + "theoretical_loss": 3.4229427383508995, + "tokens_seen": 2052980736 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843838383838384, + "loss": 1.3775, + "theoretical_loss": 3.422933674742084, + "tokens_seen": 2053046272 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038436363636363635, + "loss": 1.4104, + "theoretical_loss": 3.422924611503593, + "tokens_seen": 2053111808 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.3802541494369507, + "objective/train/avg_token_score": 0.012950729578733444, + "objective/train/docs_used": 1161710, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1741505861282349, + "objective/train/original_loss": 4.349769115447998, + "objective/train/theoretical_loss": 3.422915548635401, + "objective/train/tokens_used": 412036576, + "objective/train/unlikelihood_loss": 0.06985551118850708, + "theoretical_loss": 3.422915548635401, + "tokens_seen": 2053177344 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843434343434343, + "loss": 1.5293, + "theoretical_loss": 3.422915548635401, + "tokens_seen": 2053177344 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843232323232324, + "loss": 1.3415, + "theoretical_loss": 3.4229064861374803, + "tokens_seen": 2053242880 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038430303030303033, + "loss": 1.3631, + "theoretical_loss": 3.422897424009804, + "tokens_seen": 2053308416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842828282828283, + "loss": 1.6595, + "theoretical_loss": 3.422888362252345, + "tokens_seen": 2053373952 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038426262626262625, + "loss": 1.7031, + "theoretical_loss": 3.4228793008650773, + "tokens_seen": 2053439488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842424242424242, + "loss": 1.624, + "theoretical_loss": 3.422870239847973, + "tokens_seen": 2053505024 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038422222222222227, + "loss": 1.3284, + "theoretical_loss": 3.4228611792010053, + "tokens_seen": 2053570560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842020202020202, + "loss": 1.5913, + "theoretical_loss": 3.4228521189241476, + "tokens_seen": 2053636096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841818181818182, + "loss": 1.5343, + "theoretical_loss": 3.4228430590173726, + "tokens_seen": 2053701632 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038416161616161614, + "loss": 1.3534, + "theoretical_loss": 3.4228339994806536, + "tokens_seen": 2053767168 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038414141414141415, + "loss": 1.6541, + "theoretical_loss": 3.422824940313964, + "tokens_seen": 2053832704 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038412121212121216, + "loss": 1.2919, + "theoretical_loss": 3.422815881517276, + "tokens_seen": 2053898240 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841010101010101, + "loss": 1.2531, + "theoretical_loss": 3.4228068230905633, + "tokens_seen": 2053963776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003840808080808081, + "loss": 1.8086, + "theoretical_loss": 3.422797765033799, + "tokens_seen": 2054029312 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038406060606060603, + "loss": 1.4963, + "theoretical_loss": 3.4227887073469563, + "tokens_seen": 2054094848 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038404040404040404, + "loss": 1.3441, + "theoretical_loss": 3.422779650030008, + "tokens_seen": 2054160384 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038402020202020205, + "loss": 1.4607, + "theoretical_loss": 3.422770593082927, + "tokens_seen": 2054225920 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384, + "loss": 1.5216, + "theoretical_loss": 3.422761536505687, + "tokens_seen": 2054291456 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038397979797979797, + "loss": 1.3406, + "theoretical_loss": 3.4227524802982607, + "tokens_seen": 2054356992 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383959595959596, + "loss": 1.4556, + "theoretical_loss": 3.422743424460621, + "tokens_seen": 2054422528 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038393939393939393, + "loss": 1.1927, + "theoretical_loss": 3.4227343689927414, + "tokens_seen": 2054488064 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038391919191919195, + "loss": 1.3426, + "theoretical_loss": 3.4227253138945946, + "tokens_seen": 2054553600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838989898989899, + "loss": 1.2681, + "theoretical_loss": 3.4227162591661546, + "tokens_seen": 2054619136 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038387878787878786, + "loss": 1.3177, + "theoretical_loss": 3.422707204807393, + "tokens_seen": 2054684672 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038385858585858587, + "loss": 1.3171, + "theoretical_loss": 3.4226981508182845, + "tokens_seen": 2054750208 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.4054863154888153, + "objective/train/avg_token_score": 0.023195790126919746, + "objective/train/docs_used": 1163069, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4281989336013794, + "objective/train/original_loss": 4.372689723968506, + "objective/train/theoretical_loss": 3.422689097198802, + "objective/train/tokens_used": 413674976, + "objective/train/unlikelihood_loss": 0.06228712573647499, + "theoretical_loss": 3.422689097198802, + "tokens_seen": 2054815744 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038383838383838383, + "loss": 1.4037, + "theoretical_loss": 3.422689097198802, + "tokens_seen": 2054815744 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038381818181818184, + "loss": 1.5647, + "theoretical_loss": 3.422680043948917, + "tokens_seen": 2054881280 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837979797979798, + "loss": 1.5309, + "theoretical_loss": 3.422670991068605, + "tokens_seen": 2054946816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837777777777778, + "loss": 1.4869, + "theoretical_loss": 3.422661938557837, + "tokens_seen": 2055012352 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038375757575757576, + "loss": 1.5066, + "theoretical_loss": 3.4226528864165875, + "tokens_seen": 2055077888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837373737373737, + "loss": 1.6479, + "theoretical_loss": 3.4226438346448287, + "tokens_seen": 2055143424 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038371717171717173, + "loss": 1.2513, + "theoretical_loss": 3.422634783242535, + "tokens_seen": 2055208960 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836969696969697, + "loss": 1.4289, + "theoretical_loss": 3.422625732209678, + "tokens_seen": 2055274496 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836767676767677, + "loss": 1.3469, + "theoretical_loss": 3.4226166815462316, + "tokens_seen": 2055340032 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038365656565656566, + "loss": 1.3066, + "theoretical_loss": 3.42260763125217, + "tokens_seen": 2055405568 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836363636363636, + "loss": 1.4145, + "theoretical_loss": 3.422598581327464, + "tokens_seen": 2055471104 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836161616161616, + "loss": 1.4349, + "theoretical_loss": 3.4225895317720885, + "tokens_seen": 2055536640 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038359595959595963, + "loss": 1.3257, + "theoretical_loss": 3.4225804825860164, + "tokens_seen": 2055602176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003835757575757576, + "loss": 1.2129, + "theoretical_loss": 3.42257143376922, + "tokens_seen": 2055667712 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038355555555555555, + "loss": 1.409, + "theoretical_loss": 3.422562385321674, + "tokens_seen": 2055733248 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038353535353535356, + "loss": 1.349, + "theoretical_loss": 3.4225533372433503, + "tokens_seen": 2055798784 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003835151515151515, + "loss": 1.4454, + "theoretical_loss": 3.422544289534222, + "tokens_seen": 2055864320 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038349494949494953, + "loss": 1.4329, + "theoretical_loss": 3.422535242194263, + "tokens_seen": 2055929856 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834747474747475, + "loss": 1.3337, + "theoretical_loss": 3.4225261952234467, + "tokens_seen": 2055995392 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038345454545454544, + "loss": 1.6852, + "theoretical_loss": 3.422517148621745, + "tokens_seen": 2056060928 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038343434343434345, + "loss": 1.2157, + "theoretical_loss": 3.4225081023891324, + "tokens_seen": 2056126464 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038341414141414146, + "loss": 1.4933, + "theoretical_loss": 3.422499056525581, + "tokens_seen": 2056192000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833939393939394, + "loss": 1.2843, + "theoretical_loss": 3.4224900110310648, + "tokens_seen": 2056257536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833737373737374, + "loss": 1.3396, + "theoretical_loss": 3.422480965905556, + "tokens_seen": 2056323072 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038335353535353533, + "loss": 1.3149, + "theoretical_loss": 3.4224719211490293, + "tokens_seen": 2056388608 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.380987286567688, + "objective/train/avg_token_score": 0.0735260620713234, + "objective/train/docs_used": 1163822, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3353681564331055, + "objective/train/original_loss": 4.655409812927246, + "objective/train/theoretical_loss": 3.4224628767614567, + "objective/train/tokens_used": 415313376, + "objective/train/unlikelihood_loss": 0.04561014100909233, + "theoretical_loss": 3.4224628767614567, + "tokens_seen": 2056454144 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038333333333333334, + "loss": 1.5331, + "theoretical_loss": 3.4224628767614567, + "tokens_seen": 2056454144 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038331313131313136, + "loss": 1.4982, + "theoretical_loss": 3.4224538327428116, + "tokens_seen": 2056519680 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832929292929293, + "loss": 1.1249, + "theoretical_loss": 3.4224447890930674, + "tokens_seen": 2056585216 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038327272727272727, + "loss": 1.3073, + "theoretical_loss": 3.422435745812197, + "tokens_seen": 2056650752 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832525252525252, + "loss": 1.7152, + "theoretical_loss": 3.4224267029001743, + "tokens_seen": 2056716288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832323232323233, + "loss": 1.682, + "theoretical_loss": 3.422417660356972, + "tokens_seen": 2056781824 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038321212121212125, + "loss": 1.315, + "theoretical_loss": 3.422408618182563, + "tokens_seen": 2056847360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831919191919192, + "loss": 1.2364, + "theoretical_loss": 3.422399576376921, + "tokens_seen": 2056912896 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038317171717171716, + "loss": 1.3722, + "theoretical_loss": 3.422390534940019, + "tokens_seen": 2056978432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831515151515151, + "loss": 1.4452, + "theoretical_loss": 3.42238149387183, + "tokens_seen": 2057043968 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831313131313132, + "loss": 1.7672, + "theoretical_loss": 3.4223724531723283, + "tokens_seen": 2057109504 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038311111111111114, + "loss": 1.5932, + "theoretical_loss": 3.4223634128414853, + "tokens_seen": 2057175040 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003830909090909091, + "loss": 1.4351, + "theoretical_loss": 3.422354372879276, + "tokens_seen": 2057240576 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038307070707070706, + "loss": 1.3841, + "theoretical_loss": 3.4223453332856724, + "tokens_seen": 2057306112 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383050505050505, + "loss": 1.4025, + "theoretical_loss": 3.4223362940606483, + "tokens_seen": 2057371648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003830303030303031, + "loss": 1.4452, + "theoretical_loss": 3.422327255204177, + "tokens_seen": 2057437184 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038301010101010103, + "loss": 1.4571, + "theoretical_loss": 3.4223182167162314, + "tokens_seen": 2057502720 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382989898989899, + "loss": 1.4462, + "theoretical_loss": 3.4223091785967847, + "tokens_seen": 2057568256 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038296969696969695, + "loss": 1.4576, + "theoretical_loss": 3.4223001408458105, + "tokens_seen": 2057633792 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038294949494949496, + "loss": 1.466, + "theoretical_loss": 3.422291103463282, + "tokens_seen": 2057699328 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038292929292929297, + "loss": 1.4806, + "theoretical_loss": 3.422282066449172, + "tokens_seen": 2057764864 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829090909090909, + "loss": 1.5457, + "theoretical_loss": 3.4222730298034545, + "tokens_seen": 2057830400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828888888888889, + "loss": 1.7115, + "theoretical_loss": 3.4222639935261014, + "tokens_seen": 2057895936 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038286868686868684, + "loss": 1.4036, + "theoretical_loss": 3.4222549576170875, + "tokens_seen": 2057961472 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038284848484848485, + "loss": 1.3656, + "theoretical_loss": 3.422245922076386, + "tokens_seen": 2058027008 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.24334067106246948, + "objective/train/avg_token_score": 0.04306131601333618, + "objective/train/docs_used": 1164494, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8037949800491333, + "objective/train/original_loss": 5.774464130401611, + "objective/train/theoretical_loss": 3.4222368869039688, + "objective/train/tokens_used": 416951776, + "objective/train/unlikelihood_loss": 0.0329064279794693, + "theoretical_loss": 3.4222368869039688, + "tokens_seen": 2058092544 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038282828282828286, + "loss": 1.0842, + "theoretical_loss": 3.4222368869039688, + "tokens_seen": 2058092544 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828080808080808, + "loss": 1.4003, + "theoretical_loss": 3.42222785209981, + "tokens_seen": 2058158080 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827878787878788, + "loss": 1.5305, + "theoretical_loss": 3.4222188176638833, + "tokens_seen": 2058223616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827676767676768, + "loss": 1.5104, + "theoretical_loss": 3.422209783596161, + "tokens_seen": 2058289152 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038274747474747474, + "loss": 1.6139, + "theoretical_loss": 3.422200749896617, + "tokens_seen": 2058354688 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038272727272727276, + "loss": 1.2825, + "theoretical_loss": 3.4221917165652243, + "tokens_seen": 2058420224 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827070707070707, + "loss": 1.341, + "theoretical_loss": 3.4221826836019567, + "tokens_seen": 2058485760 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038268686868686867, + "loss": 1.5737, + "theoretical_loss": 3.4221736510067866, + "tokens_seen": 2058551296 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003826666666666667, + "loss": 1.5835, + "theoretical_loss": 3.422164618779688, + "tokens_seen": 2058616832 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038264646464646464, + "loss": 1.2643, + "theoretical_loss": 3.422155586920634, + "tokens_seen": 2058682368 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038262626262626265, + "loss": 1.3795, + "theoretical_loss": 3.4221465554295976, + "tokens_seen": 2058747904 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003826060606060606, + "loss": 1.6555, + "theoretical_loss": 3.422137524306552, + "tokens_seen": 2058813440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003825858585858586, + "loss": 1.2681, + "theoretical_loss": 3.4221284935514715, + "tokens_seen": 2058878976 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038256565656565657, + "loss": 1.2707, + "theoretical_loss": 3.4221194631643286, + "tokens_seen": 2058944512 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038254545454545453, + "loss": 1.2383, + "theoretical_loss": 3.4221104331450967, + "tokens_seen": 2059010048 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038252525252525254, + "loss": 1.2207, + "theoretical_loss": 3.4221014034937487, + "tokens_seen": 2059075584 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003825050505050505, + "loss": 1.3845, + "theoretical_loss": 3.4220923742102585, + "tokens_seen": 2059141120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824848484848485, + "loss": 1.4695, + "theoretical_loss": 3.4220833452945993, + "tokens_seen": 2059206656 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038246464646464647, + "loss": 1.3729, + "theoretical_loss": 3.4220743167467442, + "tokens_seen": 2059272192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824444444444444, + "loss": 1.438, + "theoretical_loss": 3.422065288566667, + "tokens_seen": 2059337728 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038242424242424243, + "loss": 1.547, + "theoretical_loss": 3.4220562607543403, + "tokens_seen": 2059403264 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038240404040404044, + "loss": 1.575, + "theoretical_loss": 3.422047233309738, + "tokens_seen": 2059468800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823838383838384, + "loss": 1.4872, + "theoretical_loss": 3.422038206232833, + "tokens_seen": 2059534336 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038236363636363636, + "loss": 1.4038, + "theoretical_loss": 3.422029179523599, + "tokens_seen": 2059599872 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823434343434343, + "loss": 1.4396, + "theoretical_loss": 3.422020153182009, + "tokens_seen": 2059665408 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.30645161867141724, + "objective/train/avg_token_score": 0.02268202044069767, + "objective/train/docs_used": 1165816, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.004839539527893, + "objective/train/original_loss": 4.174016952514648, + "objective/train/theoretical_loss": 3.4220111272080365, + "objective/train/tokens_used": 418590176, + "objective/train/unlikelihood_loss": 0.0708223283290863, + "theoretical_loss": 3.4220111272080365, + "tokens_seen": 2059730944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823232323232323, + "loss": 1.2435, + "theoretical_loss": 3.4220111272080365, + "tokens_seen": 2059730944 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038230303030303034, + "loss": 1.5124, + "theoretical_loss": 3.4220021016016546, + "tokens_seen": 2059796480 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822828282828283, + "loss": 1.4852, + "theoretical_loss": 3.4219930763628374, + "tokens_seen": 2059862016 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038226262626262625, + "loss": 1.3858, + "theoretical_loss": 3.421984051491557, + "tokens_seen": 2059927552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822424242424242, + "loss": 1.3737, + "theoretical_loss": 3.4219750269877878, + "tokens_seen": 2059993088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822222222222223, + "loss": 1.4172, + "theoretical_loss": 3.4219660028515024, + "tokens_seen": 2060058624 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038220202020202023, + "loss": 1.3067, + "theoretical_loss": 3.421956979082675, + "tokens_seen": 2060124160 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821818181818182, + "loss": 1.3377, + "theoretical_loss": 3.4219479556812784, + "tokens_seen": 2060189696 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038216161616161614, + "loss": 1.5462, + "theoretical_loss": 3.421938932647286, + "tokens_seen": 2060255232 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821414141414141, + "loss": 1.5533, + "theoretical_loss": 3.421929909980671, + "tokens_seen": 2060320768 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038212121212121217, + "loss": 1.5683, + "theoretical_loss": 3.4219208876814067, + "tokens_seen": 2060386304 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821010101010101, + "loss": 1.2615, + "theoretical_loss": 3.4219118657494674, + "tokens_seen": 2060451840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820808080808081, + "loss": 1.5316, + "theoretical_loss": 3.421902844184825, + "tokens_seen": 2060517376 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038206060606060604, + "loss": 1.3873, + "theoretical_loss": 3.421893822987454, + "tokens_seen": 2060582912 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038204040404040405, + "loss": 1.4241, + "theoretical_loss": 3.421884802157327, + "tokens_seen": 2060648448 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038202020202020206, + "loss": 1.4358, + "theoretical_loss": 3.4218757816944185, + "tokens_seen": 2060713984 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382, + "loss": 1.3676, + "theoretical_loss": 3.4218667615987, + "tokens_seen": 2060779520 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038197979797979797, + "loss": 1.3977, + "theoretical_loss": 3.421857741870147, + "tokens_seen": 2060845056 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038195959595959593, + "loss": 1.4248, + "theoretical_loss": 3.4218487225087317, + "tokens_seen": 2060910592 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038193939393939394, + "loss": 1.4304, + "theoretical_loss": 3.4218397035144275, + "tokens_seen": 2060976128 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038191919191919195, + "loss": 1.5699, + "theoretical_loss": 3.4218306848872073, + "tokens_seen": 2061041664 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818989898989899, + "loss": 1.3153, + "theoretical_loss": 3.421821666627046, + "tokens_seen": 2061107200 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038187878787878786, + "loss": 1.3453, + "theoretical_loss": 3.4218126487339156, + "tokens_seen": 2061172736 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818585858585859, + "loss": 1.7463, + "theoretical_loss": 3.4218036312077906, + "tokens_seen": 2061238272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818383838383839, + "loss": 1.4188, + "theoretical_loss": 3.4217946140486433, + "tokens_seen": 2061303808 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.4169110357761383, + "objective/train/avg_token_score": 0.013650353066623211, + "objective/train/docs_used": 1166525, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3619921207427979, + "objective/train/original_loss": 4.5464186668396, + "objective/train/theoretical_loss": 3.4217855972564477, + "objective/train/tokens_used": 420228576, + "objective/train/unlikelihood_loss": 0.058496687561273575, + "theoretical_loss": 3.4217855972564477, + "tokens_seen": 2061369344 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038181818181818184, + "loss": 1.3514, + "theoretical_loss": 3.4217855972564477, + "tokens_seen": 2061369344 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817979797979798, + "loss": 1.3813, + "theoretical_loss": 3.4217765808311773, + "tokens_seen": 2061434880 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038177777777777776, + "loss": 1.5357, + "theoretical_loss": 3.4217675647728054, + "tokens_seen": 2061500416 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038175757575757577, + "loss": 1.7312, + "theoretical_loss": 3.4217585490813045, + "tokens_seen": 2061565952 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817373737373738, + "loss": 1.4736, + "theoretical_loss": 3.4217495337566497, + "tokens_seen": 2061631488 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038171717171717174, + "loss": 1.5037, + "theoretical_loss": 3.4217405187988135, + "tokens_seen": 2061697024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816969696969697, + "loss": 1.4296, + "theoretical_loss": 3.421731504207769, + "tokens_seen": 2061762560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816767676767677, + "loss": 1.511, + "theoretical_loss": 3.42172248998349, + "tokens_seen": 2061828096 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038165656565656566, + "loss": 1.2112, + "theoretical_loss": 3.42171347612595, + "tokens_seen": 2061893632 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038163636363636367, + "loss": 1.6706, + "theoretical_loss": 3.4217044626351223, + "tokens_seen": 2061959168 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038161616161616163, + "loss": 1.3244, + "theoretical_loss": 3.42169544951098, + "tokens_seen": 2062024704 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815959595959596, + "loss": 1.4235, + "theoretical_loss": 3.4216864367534976, + "tokens_seen": 2062090240 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815757575757576, + "loss": 1.233, + "theoretical_loss": 3.4216774243626475, + "tokens_seen": 2062155776 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038155555555555555, + "loss": 1.2825, + "theoretical_loss": 3.4216684123384034, + "tokens_seen": 2062221312 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038153535353535357, + "loss": 1.395, + "theoretical_loss": 3.421659400680739, + "tokens_seen": 2062286848 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815151515151515, + "loss": 1.6011, + "theoretical_loss": 3.421650389389627, + "tokens_seen": 2062352384 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814949494949495, + "loss": 1.3961, + "theoretical_loss": 3.4216413784650417, + "tokens_seen": 2062417920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814747474747475, + "loss": 1.3719, + "theoretical_loss": 3.4216323679069562, + "tokens_seen": 2062483456 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038145454545454545, + "loss": 1.4384, + "theoretical_loss": 3.421623357715344, + "tokens_seen": 2062548992 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038143434343434346, + "loss": 1.4021, + "theoretical_loss": 3.421614347890178, + "tokens_seen": 2062614528 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814141414141414, + "loss": 1.5168, + "theoretical_loss": 3.4216053384314327, + "tokens_seen": 2062680064 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813939393939394, + "loss": 1.4452, + "theoretical_loss": 3.4215963293390805, + "tokens_seen": 2062745600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813737373737374, + "loss": 1.2376, + "theoretical_loss": 3.4215873206130962, + "tokens_seen": 2062811136 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038135353535353534, + "loss": 1.3614, + "theoretical_loss": 3.4215783122534518, + "tokens_seen": 2062876672 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038133333333333335, + "loss": 1.3911, + "theoretical_loss": 3.421569304260122, + "tokens_seen": 2062942208 + }, + { + "epoch": 0.24, + "objective/train/aligned_token_count": 0.5072702765464783, + "objective/train/avg_token_score": 0.02509508840739727, + "objective/train/docs_used": 1167825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6743861436843872, + "objective/train/original_loss": 4.351998329162598, + "objective/train/theoretical_loss": 3.421560296633079, + "objective/train/tokens_used": 421866976, + "objective/train/unlikelihood_loss": 0.04990775138139725, + "theoretical_loss": 3.421560296633079, + "tokens_seen": 2063007744 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813131313131313, + "loss": 1.5035, + "theoretical_loss": 3.421560296633079, + "tokens_seen": 2063007744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812929292929293, + "loss": 1.2393, + "theoretical_loss": 3.4215512893722972, + "tokens_seen": 2063073280 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812727272727273, + "loss": 1.5631, + "theoretical_loss": 3.4215422824777497, + "tokens_seen": 2063138816 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038125252525252523, + "loss": 1.2478, + "theoretical_loss": 3.42153327594941, + "tokens_seen": 2063204352 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038123232323232324, + "loss": 1.5025, + "theoretical_loss": 3.421524269787252, + "tokens_seen": 2063269888 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038121212121212125, + "loss": 1.4308, + "theoretical_loss": 3.4215152639912487, + "tokens_seen": 2063335424 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811919191919192, + "loss": 1.4696, + "theoretical_loss": 3.421506258561374, + "tokens_seen": 2063400960 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038117171717171717, + "loss": 1.3622, + "theoretical_loss": 3.421497253497601, + "tokens_seen": 2063466496 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811515151515151, + "loss": 1.4055, + "theoretical_loss": 3.4214882487999034, + "tokens_seen": 2063532032 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038113131313131314, + "loss": 1.1969, + "theoretical_loss": 3.4214792444682542, + "tokens_seen": 2063597568 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038111111111111115, + "loss": 1.3362, + "theoretical_loss": 3.421470240502628, + "tokens_seen": 2063663104 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810909090909091, + "loss": 1.4218, + "theoretical_loss": 3.4214612369029966, + "tokens_seen": 2063728640 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038107070707070706, + "loss": 1.3588, + "theoretical_loss": 3.421452233669335, + "tokens_seen": 2063794176 + }, + { + "epoch": 0.25, + "learning_rate": 0.000381050505050505, + "loss": 1.423, + "theoretical_loss": 3.421443230801616, + "tokens_seen": 2063859712 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810303030303031, + "loss": 1.4211, + "theoretical_loss": 3.421434228299814, + "tokens_seen": 2063925248 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038101010101010104, + "loss": 1.3854, + "theoretical_loss": 3.421425226163901, + "tokens_seen": 2063990784 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380989898989899, + "loss": 1.4755, + "theoretical_loss": 3.4214162243938517, + "tokens_seen": 2064056320 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038096969696969695, + "loss": 1.4164, + "theoretical_loss": 3.4214072229896395, + "tokens_seen": 2064121856 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809494949494949, + "loss": 1.4524, + "theoretical_loss": 3.4213982219512373, + "tokens_seen": 2064187392 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380929292929293, + "loss": 1.0907, + "theoretical_loss": 3.421389221278619, + "tokens_seen": 2064252928 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038090909090909093, + "loss": 1.4692, + "theoretical_loss": 3.4213802209717583, + "tokens_seen": 2064318464 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808888888888889, + "loss": 1.345, + "theoretical_loss": 3.4213712210306286, + "tokens_seen": 2064384000 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038086868686868685, + "loss": 1.4035, + "theoretical_loss": 3.421362221455203, + "tokens_seen": 2064449536 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038084848484848486, + "loss": 1.48, + "theoretical_loss": 3.4213532222454557, + "tokens_seen": 2064515072 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038082828282828287, + "loss": 1.3966, + "theoretical_loss": 3.42134422340136, + "tokens_seen": 2064580608 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.5281035900115967, + "objective/train/avg_token_score": 0.018594948574900627, + "objective/train/docs_used": 1168450, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7348947525024414, + "objective/train/original_loss": 3.9402873516082764, + "objective/train/theoretical_loss": 3.421335224922889, + "objective/train/tokens_used": 423505376, + "objective/train/unlikelihood_loss": 0.0670773833990097, + "theoretical_loss": 3.421335224922889, + "tokens_seen": 2064646144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808080808080808, + "loss": 1.7727, + "theoretical_loss": 3.421335224922889, + "tokens_seen": 2064646144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807878787878788, + "loss": 1.2763, + "theoretical_loss": 3.4213262268100166, + "tokens_seen": 2064711680 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038076767676767674, + "loss": 1.3587, + "theoretical_loss": 3.4213172290627165, + "tokens_seen": 2064777216 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038074747474747475, + "loss": 1.6083, + "theoretical_loss": 3.421308231680962, + "tokens_seen": 2064842752 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038072727272727276, + "loss": 1.5072, + "theoretical_loss": 3.421299234664727, + "tokens_seen": 2064908288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807070707070707, + "loss": 1.4557, + "theoretical_loss": 3.421290238013985, + "tokens_seen": 2064973824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806868686868687, + "loss": 1.4156, + "theoretical_loss": 3.4212812417287086, + "tokens_seen": 2065039360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806666666666667, + "loss": 1.4262, + "theoretical_loss": 3.421272245808873, + "tokens_seen": 2065104896 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038064646464646464, + "loss": 1.2626, + "theoretical_loss": 3.42126325025445, + "tokens_seen": 2065170432 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038062626262626265, + "loss": 1.5695, + "theoretical_loss": 3.4212542550654144, + "tokens_seen": 2065235968 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806060606060606, + "loss": 1.3867, + "theoretical_loss": 3.4212452602417396, + "tokens_seen": 2065301504 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038058585858585857, + "loss": 1.5888, + "theoretical_loss": 3.421236265783399, + "tokens_seen": 2065367040 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805656565656566, + "loss": 1.3008, + "theoretical_loss": 3.4212272716903653, + "tokens_seen": 2065432576 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038054545454545454, + "loss": 1.5071, + "theoretical_loss": 3.421218277962614, + "tokens_seen": 2065498112 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038052525252525255, + "loss": 1.3812, + "theoretical_loss": 3.421209284600117, + "tokens_seen": 2065563648 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805050505050505, + "loss": 1.4338, + "theoretical_loss": 3.4212002916028483, + "tokens_seen": 2065629184 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804848484848485, + "loss": 1.358, + "theoretical_loss": 3.421191298970782, + "tokens_seen": 2065694720 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038046464646464647, + "loss": 1.5437, + "theoretical_loss": 3.4211823067038907, + "tokens_seen": 2065760256 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038044444444444443, + "loss": 1.3364, + "theoretical_loss": 3.4211733148021493, + "tokens_seen": 2065825792 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038042424242424244, + "loss": 1.243, + "theoretical_loss": 3.42116432326553, + "tokens_seen": 2065891328 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038042424242424244, + "loss": 1.4324, + "theoretical_loss": 3.421155332094008, + "tokens_seen": 2065956864 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804040404040404, + "loss": 1.3529, + "theoretical_loss": 3.4211463412875553, + "tokens_seen": 2066022400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803838383838384, + "loss": 1.3293, + "theoretical_loss": 3.4211373508461462, + "tokens_seen": 2066087936 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038036363636363636, + "loss": 1.4924, + "theoretical_loss": 3.4211283607697545, + "tokens_seen": 2066153472 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803434343434343, + "loss": 1.1353, + "theoretical_loss": 3.421119371058354, + "tokens_seen": 2066219008 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.4422653913497925, + "objective/train/avg_token_score": 0.037376537919044495, + "objective/train/docs_used": 1169491, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3472920656204224, + "objective/train/original_loss": 4.558820724487305, + "objective/train/theoretical_loss": 3.421110381711917, + "objective/train/tokens_used": 425143776, + "objective/train/unlikelihood_loss": 0.06192596256732941, + "theoretical_loss": 3.421110381711917, + "tokens_seen": 2066284544 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038032323232323233, + "loss": 1.4788, + "theoretical_loss": 3.421110381711917, + "tokens_seen": 2066284544 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038030303030303034, + "loss": 1.3516, + "theoretical_loss": 3.421101392730418, + "tokens_seen": 2066350080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802828282828283, + "loss": 1.4592, + "theoretical_loss": 3.421092404113831, + "tokens_seen": 2066415616 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038026262626262626, + "loss": 1.3995, + "theoretical_loss": 3.421083415862129, + "tokens_seen": 2066481152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802424242424242, + "loss": 1.5694, + "theoretical_loss": 3.4210744279752863, + "tokens_seen": 2066546688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802222222222222, + "loss": 1.3938, + "theoretical_loss": 3.421065440453276, + "tokens_seen": 2066612224 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038020202020202024, + "loss": 1.5165, + "theoretical_loss": 3.421056453296071, + "tokens_seen": 2066677760 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801818181818182, + "loss": 1.4164, + "theoretical_loss": 3.4210474665036466, + "tokens_seen": 2066743296 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038016161616161615, + "loss": 1.5343, + "theoretical_loss": 3.421038480075975, + "tokens_seen": 2066808832 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038014141414141416, + "loss": 1.3788, + "theoretical_loss": 3.421029494013031, + "tokens_seen": 2066874368 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038012121212121217, + "loss": 1.5264, + "theoretical_loss": 3.421020508314787, + "tokens_seen": 2066939904 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038010101010101013, + "loss": 1.3608, + "theoretical_loss": 3.421011522981217, + "tokens_seen": 2067005440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800808080808081, + "loss": 1.5184, + "theoretical_loss": 3.4210025380122953, + "tokens_seen": 2067070976 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038006060606060604, + "loss": 1.351, + "theoretical_loss": 3.420993553407995, + "tokens_seen": 2067136512 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038004040404040405, + "loss": 1.5483, + "theoretical_loss": 3.42098456916829, + "tokens_seen": 2067202048 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038002020202020206, + "loss": 1.4377, + "theoretical_loss": 3.4209755852931534, + "tokens_seen": 2067267584 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038, + "loss": 1.4813, + "theoretical_loss": 3.4209666017825597, + "tokens_seen": 2067333120 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379979797979798, + "loss": 1.2604, + "theoretical_loss": 3.4209576186364816, + "tokens_seen": 2067398656 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037995959595959593, + "loss": 1.2219, + "theoretical_loss": 3.4209486358548933, + "tokens_seen": 2067464192 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379939393939394, + "loss": 1.2292, + "theoretical_loss": 3.420939653437769, + "tokens_seen": 2067529728 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037991919191919196, + "loss": 1.241, + "theoretical_loss": 3.420930671385081, + "tokens_seen": 2067595264 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798989898989899, + "loss": 1.443, + "theoretical_loss": 3.4209216896968044, + "tokens_seen": 2067660800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037987878787878787, + "loss": 1.5578, + "theoretical_loss": 3.420912708372912, + "tokens_seen": 2067726336 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037985858585858583, + "loss": 1.6159, + "theoretical_loss": 3.4209037274133776, + "tokens_seen": 2067791872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798383838383839, + "loss": 1.5248, + "theoretical_loss": 3.420894746818175, + "tokens_seen": 2067857408 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.42247068881988525, + "objective/train/avg_token_score": 0.005036116577684879, + "objective/train/docs_used": 1170265, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2853634357452393, + "objective/train/original_loss": 3.96811842918396, + "objective/train/theoretical_loss": 3.4208857665872774, + "objective/train/tokens_used": 426782176, + "objective/train/unlikelihood_loss": 0.09357985854148865, + "theoretical_loss": 3.4208857665872774, + "tokens_seen": 2067922944 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037981818181818185, + "loss": 1.4816, + "theoretical_loss": 3.4208857665872774, + "tokens_seen": 2067922944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797979797979798, + "loss": 1.4999, + "theoretical_loss": 3.4208767867206595, + "tokens_seen": 2067988480 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037977777777777776, + "loss": 1.429, + "theoretical_loss": 3.4208678072182943, + "tokens_seen": 2068054016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797575757575757, + "loss": 1.6531, + "theoretical_loss": 3.4208588280801555, + "tokens_seen": 2068119552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797373737373738, + "loss": 1.3178, + "theoretical_loss": 3.420849849306216, + "tokens_seen": 2068185088 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037971717171717174, + "loss": 1.3174, + "theoretical_loss": 3.4208408708964515, + "tokens_seen": 2068250624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796969696969697, + "loss": 1.5182, + "theoretical_loss": 3.420831892850834, + "tokens_seen": 2068316160 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037967676767676766, + "loss": 1.5146, + "theoretical_loss": 3.420822915169338, + "tokens_seen": 2068381696 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037965656565656567, + "loss": 1.5209, + "theoretical_loss": 3.4208139378519364, + "tokens_seen": 2068447232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796363636363637, + "loss": 1.2463, + "theoretical_loss": 3.4208049608986038, + "tokens_seen": 2068512768 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037961616161616163, + "loss": 1.4292, + "theoretical_loss": 3.420795984309313, + "tokens_seen": 2068578304 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795959595959596, + "loss": 1.494, + "theoretical_loss": 3.420787008084039, + "tokens_seen": 2068643840 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037957575757575755, + "loss": 1.5767, + "theoretical_loss": 3.4207780322227537, + "tokens_seen": 2068709376 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037955555555555556, + "loss": 1.4401, + "theoretical_loss": 3.4207690567254327, + "tokens_seen": 2068774912 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037953535353535357, + "loss": 1.7829, + "theoretical_loss": 3.4207600815920487, + "tokens_seen": 2068840448 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037951515151515153, + "loss": 1.3836, + "theoretical_loss": 3.4207511068225753, + "tokens_seen": 2068905984 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794949494949495, + "loss": 1.5816, + "theoretical_loss": 3.4207421324169864, + "tokens_seen": 2068971520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794747474747475, + "loss": 1.3778, + "theoretical_loss": 3.4207331583752563, + "tokens_seen": 2069037056 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037945454545454545, + "loss": 1.3386, + "theoretical_loss": 3.4207241846973577, + "tokens_seen": 2069102592 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037943434343434346, + "loss": 1.5548, + "theoretical_loss": 3.420715211383265, + "tokens_seen": 2069168128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794141414141414, + "loss": 1.4954, + "theoretical_loss": 3.4207062384329516, + "tokens_seen": 2069233664 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793939393939394, + "loss": 1.2441, + "theoretical_loss": 3.4206972658463917, + "tokens_seen": 2069299200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793737373737374, + "loss": 1.5984, + "theoretical_loss": 3.420688293623559, + "tokens_seen": 2069364736 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037935353535353535, + "loss": 1.3587, + "theoretical_loss": 3.4206793217644265, + "tokens_seen": 2069430272 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037933333333333336, + "loss": 1.3259, + "theoretical_loss": 3.420670350268969, + "tokens_seen": 2069495808 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.39571115374565125, + "objective/train/avg_token_score": 0.015201877802610397, + "objective/train/docs_used": 1170942, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3105769157409668, + "objective/train/original_loss": 4.975223064422607, + "objective/train/theoretical_loss": 3.420661379137159, + "objective/train/tokens_used": 428420576, + "objective/train/unlikelihood_loss": 0.05080878734588623, + "theoretical_loss": 3.420661379137159, + "tokens_seen": 2069561344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793131313131313, + "loss": 1.4032, + "theoretical_loss": 3.420661379137159, + "tokens_seen": 2069561344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792929292929293, + "loss": 1.3405, + "theoretical_loss": 3.420652408368971, + "tokens_seen": 2069626880 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792727272727273, + "loss": 1.4213, + "theoretical_loss": 3.4206434379643786, + "tokens_seen": 2069692416 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037925252525252524, + "loss": 1.3388, + "theoretical_loss": 3.420634467923356, + "tokens_seen": 2069757952 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037923232323232325, + "loss": 1.2898, + "theoretical_loss": 3.420625498245876, + "tokens_seen": 2069823488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792121212121212, + "loss": 1.3475, + "theoretical_loss": 3.4206165289319133, + "tokens_seen": 2069889024 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791919191919192, + "loss": 1.4743, + "theoretical_loss": 3.420607559981441, + "tokens_seen": 2069954560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791717171717172, + "loss": 1.279, + "theoretical_loss": 3.420598591394433, + "tokens_seen": 2070020096 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037915151515151513, + "loss": 1.5811, + "theoretical_loss": 3.420589623170864, + "tokens_seen": 2070085632 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037913131313131314, + "loss": 1.4274, + "theoretical_loss": 3.4205806553107063, + "tokens_seen": 2070151168 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037911111111111115, + "loss": 1.2846, + "theoretical_loss": 3.4205716878139345, + "tokens_seen": 2070216704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790909090909091, + "loss": 1.4081, + "theoretical_loss": 3.420562720680522, + "tokens_seen": 2070282240 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037907070707070707, + "loss": 1.6356, + "theoretical_loss": 3.420553753910443, + "tokens_seen": 2070347776 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379050505050505, + "loss": 1.512, + "theoretical_loss": 3.4205447875036707, + "tokens_seen": 2070413312 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037903030303030303, + "loss": 1.4141, + "theoretical_loss": 3.42053582146018, + "tokens_seen": 2070478848 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037901010101010105, + "loss": 1.393, + "theoretical_loss": 3.4205268557799435, + "tokens_seen": 2070544384 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378989898989899, + "loss": 1.5733, + "theoretical_loss": 3.420517890462935, + "tokens_seen": 2070609920 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037896969696969696, + "loss": 1.1883, + "theoretical_loss": 3.420508925509129, + "tokens_seen": 2070675456 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789494949494949, + "loss": 1.4328, + "theoretical_loss": 3.420499960918499, + "tokens_seen": 2070740992 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378929292929293, + "loss": 1.3877, + "theoretical_loss": 3.420490996691019, + "tokens_seen": 2070806528 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037890909090909094, + "loss": 1.5501, + "theoretical_loss": 3.420482032826662, + "tokens_seen": 2070872064 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003788888888888889, + "loss": 1.4109, + "theoretical_loss": 3.4204730693254035, + "tokens_seen": 2070937600 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037886868686868685, + "loss": 1.3661, + "theoretical_loss": 3.4204641061872154, + "tokens_seen": 2071003136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003788484848484848, + "loss": 1.4302, + "theoretical_loss": 3.420455143412072, + "tokens_seen": 2071068672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003788282828282829, + "loss": 1.4722, + "theoretical_loss": 3.4204461809999476, + "tokens_seen": 2071134208 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.4280914068222046, + "objective/train/avg_token_score": 0.011312766931951046, + "objective/train/docs_used": 1172086, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4044407606124878, + "objective/train/original_loss": 3.66874361038208, + "objective/train/theoretical_loss": 3.4204372189508163, + "objective/train/tokens_used": 430058976, + "objective/train/unlikelihood_loss": 0.0882360190153122, + "theoretical_loss": 3.4204372189508163, + "tokens_seen": 2071199744 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037880808080808083, + "loss": 1.3167, + "theoretical_loss": 3.4204372189508163, + "tokens_seen": 2071199744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787878787878788, + "loss": 1.4364, + "theoretical_loss": 3.4204282572646507, + "tokens_seen": 2071265280 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037876767676767674, + "loss": 1.5054, + "theoretical_loss": 3.420419295941426, + "tokens_seen": 2071330816 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037874747474747476, + "loss": 1.5036, + "theoretical_loss": 3.4204103349811152, + "tokens_seen": 2071396352 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037872727272727277, + "loss": 1.4788, + "theoretical_loss": 3.420401374383692, + "tokens_seen": 2071461888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787070707070707, + "loss": 1.2752, + "theoretical_loss": 3.4203924141491306, + "tokens_seen": 2071527424 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786868686868687, + "loss": 1.209, + "theoretical_loss": 3.4203834542774043, + "tokens_seen": 2071592960 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037866666666666664, + "loss": 1.4583, + "theoretical_loss": 3.420374494768488, + "tokens_seen": 2071658496 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037864646464646465, + "loss": 1.3893, + "theoretical_loss": 3.420365535622355, + "tokens_seen": 2071724032 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037862626262626266, + "loss": 1.3295, + "theoretical_loss": 3.4203565768389783, + "tokens_seen": 2071789568 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786060606060606, + "loss": 1.4072, + "theoretical_loss": 3.420347618418333, + "tokens_seen": 2071855104 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785858585858586, + "loss": 1.4761, + "theoretical_loss": 3.4203386603603922, + "tokens_seen": 2071920640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785656565656566, + "loss": 1.4877, + "theoretical_loss": 3.4203297026651303, + "tokens_seen": 2071986176 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037854545454545454, + "loss": 1.5255, + "theoretical_loss": 3.4203207453325204, + "tokens_seen": 2072051712 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037852525252525255, + "loss": 1.2721, + "theoretical_loss": 3.4203117883625365, + "tokens_seen": 2072117248 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785050505050505, + "loss": 1.3668, + "theoretical_loss": 3.420302831755153, + "tokens_seen": 2072182784 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037848484848484847, + "loss": 1.4005, + "theoretical_loss": 3.4202938755103434, + "tokens_seen": 2072248320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784646464646465, + "loss": 1.4546, + "theoretical_loss": 3.420284919628082, + "tokens_seen": 2072313856 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784444444444445, + "loss": 1.5243, + "theoretical_loss": 3.4202759641083413, + "tokens_seen": 2072379392 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037842424242424244, + "loss": 1.0941, + "theoretical_loss": 3.4202670089510967, + "tokens_seen": 2072444928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784040404040404, + "loss": 1.1452, + "theoretical_loss": 3.4202580541563217, + "tokens_seen": 2072510464 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783838383838384, + "loss": 1.349, + "theoretical_loss": 3.4202490997239896, + "tokens_seen": 2072576000 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037836363636363637, + "loss": 1.4917, + "theoretical_loss": 3.4202401456540747, + "tokens_seen": 2072641536 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783434343434344, + "loss": 1.5066, + "theoretical_loss": 3.420231191946551, + "tokens_seen": 2072707072 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037832323232323234, + "loss": 1.5049, + "theoretical_loss": 3.420222238601392, + "tokens_seen": 2072772608 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.39668866991996765, + "objective/train/avg_token_score": 0.006178613286465406, + "objective/train/docs_used": 1172787, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4482109546661377, + "objective/train/original_loss": 4.506319522857666, + "objective/train/theoretical_loss": 3.420213285618572, + "objective/train/tokens_used": 431697376, + "objective/train/unlikelihood_loss": 0.05467967316508293, + "theoretical_loss": 3.420213285618572, + "tokens_seen": 2072838144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783030303030303, + "loss": 1.2255, + "theoretical_loss": 3.420213285618572, + "tokens_seen": 2072838144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782828282828283, + "loss": 1.4164, + "theoretical_loss": 3.420204332998064, + "tokens_seen": 2072903680 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037826262626262626, + "loss": 1.386, + "theoretical_loss": 3.420195380739843, + "tokens_seen": 2072969216 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782424242424243, + "loss": 1.3431, + "theoretical_loss": 3.4201864288438824, + "tokens_seen": 2073034752 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037822222222222223, + "loss": 1.4041, + "theoretical_loss": 3.4201774773101556, + "tokens_seen": 2073100288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782020202020202, + "loss": 1.277, + "theoretical_loss": 3.4201685261386374, + "tokens_seen": 2073165824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781818181818182, + "loss": 1.3984, + "theoretical_loss": 3.4201595753293015, + "tokens_seen": 2073231360 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037816161616161615, + "loss": 1.358, + "theoretical_loss": 3.420150624882121, + "tokens_seen": 2073296896 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037814141414141417, + "loss": 1.3982, + "theoretical_loss": 3.4201416747970708, + "tokens_seen": 2073362432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781212121212121, + "loss": 1.4702, + "theoretical_loss": 3.4201327250741245, + "tokens_seen": 2073427968 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037810101010101013, + "loss": 1.4477, + "theoretical_loss": 3.4201237757132557, + "tokens_seen": 2073493504 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003780808080808081, + "loss": 1.442, + "theoretical_loss": 3.420114826714438, + "tokens_seen": 2073559040 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037806060606060605, + "loss": 1.3965, + "theoretical_loss": 3.4201058780776465, + "tokens_seen": 2073624576 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037804040404040406, + "loss": 1.4009, + "theoretical_loss": 3.4200969298028543, + "tokens_seen": 2073690112 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378020202020202, + "loss": 1.5525, + "theoretical_loss": 3.4200879818900347, + "tokens_seen": 2073755648 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378, + "loss": 1.5832, + "theoretical_loss": 3.420079034339163, + "tokens_seen": 2073821184 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377979797979798, + "loss": 1.3648, + "theoretical_loss": 3.4200700871502123, + "tokens_seen": 2073886720 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037795959595959594, + "loss": 1.6466, + "theoretical_loss": 3.420061140323157, + "tokens_seen": 2073952256 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037793939393939395, + "loss": 1.3599, + "theoretical_loss": 3.4200521938579707, + "tokens_seen": 2074017792 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037791919191919196, + "loss": 1.2817, + "theoretical_loss": 3.420043247754627, + "tokens_seen": 2074083328 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778989898989899, + "loss": 1.3824, + "theoretical_loss": 3.4200343020131, + "tokens_seen": 2074148864 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778787878787879, + "loss": 1.3301, + "theoretical_loss": 3.4200253566333645, + "tokens_seen": 2074214400 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037785858585858583, + "loss": 1.4708, + "theoretical_loss": 3.4200164116153937, + "tokens_seen": 2074279936 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037783838383838384, + "loss": 1.2739, + "theoretical_loss": 3.420007466959161, + "tokens_seen": 2074345472 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037781818181818186, + "loss": 1.3532, + "theoretical_loss": 3.419998522664641, + "tokens_seen": 2074411008 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.3930840790271759, + "objective/train/avg_token_score": 0.019307734444737434, + "objective/train/docs_used": 1174027, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3908665180206299, + "objective/train/original_loss": 3.919964551925659, + "objective/train/theoretical_loss": 3.4199895787318084, + "objective/train/tokens_used": 433335776, + "objective/train/unlikelihood_loss": 0.07689140737056732, + "theoretical_loss": 3.4199895787318084, + "tokens_seen": 2074476544 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777979797979798, + "loss": 1.4928, + "theoretical_loss": 3.4199895787318084, + "tokens_seen": 2074476544 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037777777777777777, + "loss": 1.4619, + "theoretical_loss": 3.4199806351606354, + "tokens_seen": 2074542080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777575757575757, + "loss": 1.4482, + "theoretical_loss": 3.4199716919510976, + "tokens_seen": 2074607616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777373737373738, + "loss": 1.2923, + "theoretical_loss": 3.4199627491031674, + "tokens_seen": 2074673152 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037771717171717175, + "loss": 1.1326, + "theoretical_loss": 3.41995380661682, + "tokens_seen": 2074738688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776969696969697, + "loss": 1.6064, + "theoretical_loss": 3.419944864492029, + "tokens_seen": 2074804224 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037767676767676766, + "loss": 1.3603, + "theoretical_loss": 3.4199359227287687, + "tokens_seen": 2074869760 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776565656565656, + "loss": 1.453, + "theoretical_loss": 3.4199269813270123, + "tokens_seen": 2074935296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776363636363637, + "loss": 1.5569, + "theoretical_loss": 3.419918040286734, + "tokens_seen": 2075000832 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037761616161616164, + "loss": 1.8289, + "theoretical_loss": 3.4199090996079082, + "tokens_seen": 2075066368 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775959595959596, + "loss": 1.3843, + "theoretical_loss": 3.419900159290509, + "tokens_seen": 2075131904 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037757575757575755, + "loss": 1.5317, + "theoretical_loss": 3.419891219334509, + "tokens_seen": 2075197440 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037755555555555557, + "loss": 1.4498, + "theoretical_loss": 3.419882279739884, + "tokens_seen": 2075262976 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775353535353536, + "loss": 1.4489, + "theoretical_loss": 3.4198733405066064, + "tokens_seen": 2075328512 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037751515151515153, + "loss": 1.501, + "theoretical_loss": 3.4198644016346513, + "tokens_seen": 2075394048 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003774949494949495, + "loss": 1.3273, + "theoretical_loss": 3.4198554631239926, + "tokens_seen": 2075459584 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037747474747474745, + "loss": 1.3192, + "theoretical_loss": 3.4198465249746035, + "tokens_seen": 2075525120 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037745454545454546, + "loss": 1.4783, + "theoretical_loss": 3.419837587186459, + "tokens_seen": 2075590656 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037743434343434347, + "loss": 1.3299, + "theoretical_loss": 3.419828649759532, + "tokens_seen": 2075656192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003774141414141414, + "loss": 1.6143, + "theoretical_loss": 3.419819712693797, + "tokens_seen": 2075721728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773939393939394, + "loss": 1.337, + "theoretical_loss": 3.4198107759892284, + "tokens_seen": 2075787264 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773737373737374, + "loss": 1.3659, + "theoretical_loss": 3.4198018396457996, + "tokens_seen": 2075852800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037735353535353535, + "loss": 1.3633, + "theoretical_loss": 3.4197929036634855, + "tokens_seen": 2075918336 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037733333333333336, + "loss": 1.5404, + "theoretical_loss": 3.4197839680422586, + "tokens_seen": 2075983872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773131313131313, + "loss": 1.6373, + "theoretical_loss": 3.419775032782095, + "tokens_seen": 2076049408 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.4901026487350464, + "objective/train/avg_token_score": 0.014933989383280277, + "objective/train/docs_used": 1174812, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6007699966430664, + "objective/train/original_loss": 4.3715410232543945, + "objective/train/theoretical_loss": 3.4197660978829667, + "objective/train/tokens_used": 434974176, + "objective/train/unlikelihood_loss": 0.044615305960178375, + "theoretical_loss": 3.4197660978829667, + "tokens_seen": 2076114944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772929292929293, + "loss": 1.4418, + "theoretical_loss": 3.4197660978829667, + "tokens_seen": 2076114944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772727272727273, + "loss": 1.5686, + "theoretical_loss": 3.4197571633448485, + "tokens_seen": 2076180480 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037725252525252524, + "loss": 1.462, + "theoretical_loss": 3.4197482291677144, + "tokens_seen": 2076246016 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037723232323232325, + "loss": 1.3599, + "theoretical_loss": 3.4197392953515386, + "tokens_seen": 2076311552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772121212121212, + "loss": 1.2252, + "theoretical_loss": 3.419730361896295, + "tokens_seen": 2076377088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771919191919192, + "loss": 1.352, + "theoretical_loss": 3.4197214288019575, + "tokens_seen": 2076442624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771717171717172, + "loss": 1.349, + "theoretical_loss": 3.4197124960685006, + "tokens_seen": 2076508160 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037715151515151514, + "loss": 1.5102, + "theoretical_loss": 3.419703563695898, + "tokens_seen": 2076573696 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037713131313131315, + "loss": 1.2434, + "theoretical_loss": 3.4196946316841235, + "tokens_seen": 2076639232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771111111111111, + "loss": 1.624, + "theoretical_loss": 3.419685700033151, + "tokens_seen": 2076704768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770909090909091, + "loss": 1.3274, + "theoretical_loss": 3.4196767687429555, + "tokens_seen": 2076770304 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037707070707070707, + "loss": 1.3486, + "theoretical_loss": 3.41966783781351, + "tokens_seen": 2076835840 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037705050505050503, + "loss": 1.4912, + "theoretical_loss": 3.4196589072447896, + "tokens_seen": 2076901376 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037703030303030304, + "loss": 1.5207, + "theoretical_loss": 3.4196499770367668, + "tokens_seen": 2076966912 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037701010101010105, + "loss": 1.4682, + "theoretical_loss": 3.419641047189417, + "tokens_seen": 2077032448 + }, + { + "epoch": 0.25, + "learning_rate": 0.000376989898989899, + "loss": 1.3966, + "theoretical_loss": 3.4196321177027142, + "tokens_seen": 2077097984 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037696969696969696, + "loss": 1.4683, + "theoretical_loss": 3.4196231885766317, + "tokens_seen": 2077163520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003769494949494949, + "loss": 1.3407, + "theoretical_loss": 3.4196142598111443, + "tokens_seen": 2077229056 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037692929292929293, + "loss": 1.4778, + "theoretical_loss": 3.4196053314062254, + "tokens_seen": 2077294592 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037690909090909094, + "loss": 1.3302, + "theoretical_loss": 3.419596403361849, + "tokens_seen": 2077360128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768888888888889, + "loss": 1.2834, + "theoretical_loss": 3.4195874756779903, + "tokens_seen": 2077425664 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037686868686868686, + "loss": 1.4535, + "theoretical_loss": 3.4195785483546226, + "tokens_seen": 2077491200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768484848484848, + "loss": 1.4252, + "theoretical_loss": 3.4195696213917195, + "tokens_seen": 2077556736 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768282828282829, + "loss": 1.3688, + "theoretical_loss": 3.419560694789256, + "tokens_seen": 2077622272 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037680808080808084, + "loss": 1.6246, + "theoretical_loss": 3.419551768547205, + "tokens_seen": 2077687808 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.33198925852775574, + "objective/train/avg_token_score": 0.010006610304117203, + "objective/train/docs_used": 1175928, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0175628662109375, + "objective/train/original_loss": 4.5042290687561035, + "objective/train/theoretical_loss": 3.419542842665542, + "objective/train/tokens_used": 436612576, + "objective/train/unlikelihood_loss": 0.040932752192020416, + "theoretical_loss": 3.419542842665542, + "tokens_seen": 2077753344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767878787878788, + "loss": 1.4852, + "theoretical_loss": 3.419542842665542, + "tokens_seen": 2077753344 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037676767676767675, + "loss": 1.6318, + "theoretical_loss": 3.41953391714424, + "tokens_seen": 2077818880 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767474747474747, + "loss": 1.4043, + "theoretical_loss": 3.4195249919832738, + "tokens_seen": 2077884416 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037672727272727277, + "loss": 1.6004, + "theoretical_loss": 3.419516067182617, + "tokens_seen": 2077949952 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037670707070707073, + "loss": 1.3982, + "theoretical_loss": 3.419507142742244, + "tokens_seen": 2078015488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766868686868687, + "loss": 1.2151, + "theoretical_loss": 3.4194982186621288, + "tokens_seen": 2078081024 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037666666666666664, + "loss": 1.4042, + "theoretical_loss": 3.4194892949422453, + "tokens_seen": 2078146560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766464646464647, + "loss": 1.372, + "theoretical_loss": 3.419480371582568, + "tokens_seen": 2078212096 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037662626262626266, + "loss": 1.5022, + "theoretical_loss": 3.419471448583071, + "tokens_seen": 2078277632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766060606060606, + "loss": 1.4701, + "theoretical_loss": 3.419462525943727, + "tokens_seen": 2078343168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765858585858586, + "loss": 1.4062, + "theoretical_loss": 3.419453603664512, + "tokens_seen": 2078408704 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037656565656565654, + "loss": 1.4994, + "theoretical_loss": 3.4194446817453996, + "tokens_seen": 2078474240 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765454545454546, + "loss": 1.456, + "theoretical_loss": 3.4194357601863636, + "tokens_seen": 2078539776 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037652525252525256, + "loss": 1.5474, + "theoretical_loss": 3.419426838987378, + "tokens_seen": 2078605312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765050505050505, + "loss": 1.5788, + "theoretical_loss": 3.419417918148417, + "tokens_seen": 2078670848 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037648484848484847, + "loss": 1.3919, + "theoretical_loss": 3.4194089976694553, + "tokens_seen": 2078736384 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037646464646464643, + "loss": 1.3453, + "theoretical_loss": 3.419400077550466, + "tokens_seen": 2078801920 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003764444444444445, + "loss": 1.4328, + "theoretical_loss": 3.4193911577914244, + "tokens_seen": 2078867456 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037642424242424245, + "loss": 1.3432, + "theoretical_loss": 3.419382238392304, + "tokens_seen": 2078932992 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003764040404040404, + "loss": 1.4633, + "theoretical_loss": 3.4193733193530784, + "tokens_seen": 2078998528 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037638383838383836, + "loss": 1.4656, + "theoretical_loss": 3.419364400673723, + "tokens_seen": 2079064064 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763636363636364, + "loss": 1.5146, + "theoretical_loss": 3.4193554823542107, + "tokens_seen": 2079129600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763434343434344, + "loss": 1.6026, + "theoretical_loss": 3.419346564394516, + "tokens_seen": 2079195136 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037632323232323234, + "loss": 1.3136, + "theoretical_loss": 3.419337646794614, + "tokens_seen": 2079260672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763030303030303, + "loss": 1.374, + "theoretical_loss": 3.4193287295544774, + "tokens_seen": 2079326208 + }, + { + "epoch": 0.25, + "objective/train/aligned_token_count": 0.49841153621673584, + "objective/train/avg_token_score": 0.009859995916485786, + "objective/train/docs_used": 1176572, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5007386207580566, + "objective/train/original_loss": 3.7215120792388916, + "objective/train/theoretical_loss": 3.419319812674081, + "objective/train/tokens_used": 438250976, + "objective/train/unlikelihood_loss": 0.09313442558050156, + "theoretical_loss": 3.419319812674081, + "tokens_seen": 2079391744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037628282828282826, + "loss": 1.6772, + "theoretical_loss": 3.419319812674081, + "tokens_seen": 2079391744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037626262626262627, + "loss": 1.4377, + "theoretical_loss": 3.4193108961533993, + "tokens_seen": 2079457280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003762424242424243, + "loss": 1.5206, + "theoretical_loss": 3.4193019799924063, + "tokens_seen": 2079522816 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037622222222222224, + "loss": 1.4173, + "theoretical_loss": 3.4192930641910753, + "tokens_seen": 2079588352 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003762020202020202, + "loss": 1.2782, + "theoretical_loss": 3.4192841487493815, + "tokens_seen": 2079653888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003761818181818182, + "loss": 1.2694, + "theoretical_loss": 3.4192752336672987, + "tokens_seen": 2079719424 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037616161616161616, + "loss": 1.5735, + "theoretical_loss": 3.4192663189448007, + "tokens_seen": 2079784960 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037614141414141417, + "loss": 1.4436, + "theoretical_loss": 3.4192574045818622, + "tokens_seen": 2079850496 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037612121212121213, + "loss": 1.3291, + "theoretical_loss": 3.4192484905784575, + "tokens_seen": 2079916032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003761010101010101, + "loss": 1.6037, + "theoretical_loss": 3.41923957693456, + "tokens_seen": 2079981568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760808080808081, + "loss": 1.3677, + "theoretical_loss": 3.4192306636501444, + "tokens_seen": 2080047104 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037606060606060605, + "loss": 1.4373, + "theoretical_loss": 3.4192217507251854, + "tokens_seen": 2080112640 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037604040404040406, + "loss": 1.397, + "theoretical_loss": 3.419212838159656, + "tokens_seen": 2080178176 + }, + { + "epoch": 0.26, + "learning_rate": 0.000376020202020202, + "loss": 1.429, + "theoretical_loss": 3.4192039259535307, + "tokens_seen": 2080243712 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037600000000000003, + "loss": 1.2791, + "theoretical_loss": 3.4191950141067844, + "tokens_seen": 2080309248 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375979797979798, + "loss": 1.2927, + "theoretical_loss": 3.419186102619391, + "tokens_seen": 2080374784 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037595959595959595, + "loss": 1.4066, + "theoretical_loss": 3.419177191491324, + "tokens_seen": 2080440320 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037593939393939396, + "loss": 1.522, + "theoretical_loss": 3.4191682807225585, + "tokens_seen": 2080505856 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003759191919191919, + "loss": 1.5626, + "theoretical_loss": 3.419159370313068, + "tokens_seen": 2080571392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758989898989899, + "loss": 1.4265, + "theoretical_loss": 3.419150460262827, + "tokens_seen": 2080636928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758787878787879, + "loss": 1.4347, + "theoretical_loss": 3.41914155057181, + "tokens_seen": 2080702464 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037585858585858584, + "loss": 1.4526, + "theoretical_loss": 3.419132641239991, + "tokens_seen": 2080768000 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037583838383838385, + "loss": 1.5855, + "theoretical_loss": 3.419123732267344, + "tokens_seen": 2080833536 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037581818181818186, + "loss": 1.2248, + "theoretical_loss": 3.4191148236538433, + "tokens_seen": 2080899072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757979797979798, + "loss": 1.4762, + "theoretical_loss": 3.419105915399463, + "tokens_seen": 2080964608 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.43218475580215454, + "objective/train/avg_token_score": 0.01707909256219864, + "objective/train/docs_used": 1177438, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4051634073257446, + "objective/train/original_loss": 4.354172229766846, + "objective/train/theoretical_loss": 3.4190970075041776, + "objective/train/tokens_used": 439889376, + "objective/train/unlikelihood_loss": 0.06684666872024536, + "theoretical_loss": 3.4190970075041776, + "tokens_seen": 2081030144 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757777777777778, + "loss": 1.4901, + "theoretical_loss": 3.4190970075041776, + "tokens_seen": 2081030144 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037575757575757573, + "loss": 1.3864, + "theoretical_loss": 3.4190880999679614, + "tokens_seen": 2081095680 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037573737373737374, + "loss": 1.3233, + "theoretical_loss": 3.4190791927907878, + "tokens_seen": 2081161216 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037571717171717175, + "loss": 1.5315, + "theoretical_loss": 3.419070285972632, + "tokens_seen": 2081226752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003756969696969697, + "loss": 1.3804, + "theoretical_loss": 3.419061379513468, + "tokens_seen": 2081292288 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037567676767676767, + "loss": 1.3739, + "theoretical_loss": 3.4190524734132692, + "tokens_seen": 2081357824 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003756565656565656, + "loss": 1.5024, + "theoretical_loss": 3.419043567672011, + "tokens_seen": 2081423360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003756363636363637, + "loss": 1.3044, + "theoretical_loss": 3.4190346622896675, + "tokens_seen": 2081488896 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037561616161616165, + "loss": 1.5856, + "theoretical_loss": 3.419025757266212, + "tokens_seen": 2081554432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755959595959596, + "loss": 1.494, + "theoretical_loss": 3.4190168526016196, + "tokens_seen": 2081619968 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037557575757575756, + "loss": 1.2753, + "theoretical_loss": 3.4190079482958637, + "tokens_seen": 2081685504 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755555555555555, + "loss": 1.477, + "theoretical_loss": 3.4189990443489195, + "tokens_seen": 2081751040 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755353535353536, + "loss": 1.5872, + "theoretical_loss": 3.4189901407607612, + "tokens_seen": 2081816576 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037551515151515154, + "loss": 1.6336, + "theoretical_loss": 3.418981237531362, + "tokens_seen": 2081882112 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754949494949495, + "loss": 1.4524, + "theoretical_loss": 3.4189723346606975, + "tokens_seen": 2081947648 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037547474747474745, + "loss": 1.4721, + "theoretical_loss": 3.4189634321487405, + "tokens_seen": 2082013184 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037545454545454546, + "loss": 1.4003, + "theoretical_loss": 3.4189545299954665, + "tokens_seen": 2082078720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754343434343435, + "loss": 1.5892, + "theoretical_loss": 3.4189456282008495, + "tokens_seen": 2082144256 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037541414141414143, + "loss": 1.4227, + "theoretical_loss": 3.4189367267648634, + "tokens_seen": 2082209792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753939393939394, + "loss": 1.4975, + "theoretical_loss": 3.4189278256874824, + "tokens_seen": 2082275328 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037537373737373735, + "loss": 1.6269, + "theoretical_loss": 3.4189189249686813, + "tokens_seen": 2082340864 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037535353535353536, + "loss": 1.3594, + "theoretical_loss": 3.418910024608434, + "tokens_seen": 2082406400 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037533333333333337, + "loss": 1.3369, + "theoretical_loss": 3.4189011246067147, + "tokens_seen": 2082471936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753131313131313, + "loss": 1.4857, + "theoretical_loss": 3.418892224963498, + "tokens_seen": 2082537472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752929292929293, + "loss": 1.3987, + "theoretical_loss": 3.418883325678758, + "tokens_seen": 2082603008 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.34909579157829285, + "objective/train/avg_token_score": 0.03110983781516552, + "objective/train/docs_used": 1177725, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2431418895721436, + "objective/train/original_loss": 4.56602668762207, + "objective/train/theoretical_loss": 3.4188744267524687, + "objective/train/tokens_used": 441527776, + "objective/train/unlikelihood_loss": 0.06893003731966019, + "theoretical_loss": 3.4188744267524687, + "tokens_seen": 2082668544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752727272727273, + "loss": 1.4113, + "theoretical_loss": 3.4188744267524687, + "tokens_seen": 2082668544 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037525252525252525, + "loss": 1.5249, + "theoretical_loss": 3.4188655281846048, + "tokens_seen": 2082734080 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037523232323232326, + "loss": 1.6509, + "theoretical_loss": 3.4188566299751404, + "tokens_seen": 2082799616 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752121212121212, + "loss": 1.4777, + "theoretical_loss": 3.41884773212405, + "tokens_seen": 2082865152 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003751919191919192, + "loss": 1.463, + "theoretical_loss": 3.418838834631307, + "tokens_seen": 2082930688 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003751717171717172, + "loss": 1.6285, + "theoretical_loss": 3.4188299374968874, + "tokens_seen": 2082996224 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037515151515151514, + "loss": 1.4813, + "theoretical_loss": 3.418821040720764, + "tokens_seen": 2083061760 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037513131313131315, + "loss": 1.7179, + "theoretical_loss": 3.418812144302912, + "tokens_seen": 2083127296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003751111111111111, + "loss": 1.5771, + "theoretical_loss": 3.418803248243305, + "tokens_seen": 2083192832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750909090909091, + "loss": 1.6911, + "theoretical_loss": 3.4187943525419175, + "tokens_seen": 2083258368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750707070707071, + "loss": 1.4178, + "theoretical_loss": 3.4187854571987244, + "tokens_seen": 2083323904 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037505050505050503, + "loss": 1.5065, + "theoretical_loss": 3.418776562213699, + "tokens_seen": 2083389440 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037503030303030305, + "loss": 1.4285, + "theoretical_loss": 3.418767667586817, + "tokens_seen": 2083454976 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375010101010101, + "loss": 1.4658, + "theoretical_loss": 3.418758773318051, + "tokens_seen": 2083520512 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374989898989899, + "loss": 1.5077, + "theoretical_loss": 3.4187498794073763, + "tokens_seen": 2083586048 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037496969696969697, + "loss": 1.6018, + "theoretical_loss": 3.4187409858547673, + "tokens_seen": 2083651584 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374949494949495, + "loss": 1.4844, + "theoretical_loss": 3.418732092660198, + "tokens_seen": 2083717120 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037492929292929294, + "loss": 1.2961, + "theoretical_loss": 3.418723199823643, + "tokens_seen": 2083782656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749090909090909, + "loss": 1.7236, + "theoretical_loss": 3.418714307345076, + "tokens_seen": 2083848192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003748888888888889, + "loss": 1.4139, + "theoretical_loss": 3.4187054152244727, + "tokens_seen": 2083913728 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037486868686868686, + "loss": 1.3001, + "theoretical_loss": 3.418696523461806, + "tokens_seen": 2083979264 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003748484848484849, + "loss": 1.4182, + "theoretical_loss": 3.4186876320570505, + "tokens_seen": 2084044800 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037482828282828283, + "loss": 1.4754, + "theoretical_loss": 3.4186787410101815, + "tokens_seen": 2084110336 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037480808080808084, + "loss": 1.3918, + "theoretical_loss": 3.418669850321172, + "tokens_seen": 2084175872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747878787878788, + "loss": 1.3437, + "theoretical_loss": 3.4186609599899977, + "tokens_seen": 2084241408 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.3273460268974304, + "objective/train/avg_token_score": 0.024734986945986748, + "objective/train/docs_used": 1178778, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1921221017837524, + "objective/train/original_loss": 4.915249347686768, + "objective/train/theoretical_loss": 3.418652070016632, + "objective/train/tokens_used": 443166176, + "objective/train/unlikelihood_loss": 0.03642764315009117, + "theoretical_loss": 3.418652070016632, + "tokens_seen": 2084306944 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037476767676767676, + "loss": 1.6279, + "theoretical_loss": 3.418652070016632, + "tokens_seen": 2084306944 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037474747474747477, + "loss": 1.4763, + "theoretical_loss": 3.418643180401049, + "tokens_seen": 2084372480 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747272727272727, + "loss": 1.4031, + "theoretical_loss": 3.418634291143224, + "tokens_seen": 2084438016 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037470707070707073, + "loss": 1.3153, + "theoretical_loss": 3.418625402243131, + "tokens_seen": 2084503552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003746868686868687, + "loss": 1.624, + "theoretical_loss": 3.4186165137007443, + "tokens_seen": 2084569088 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037466666666666665, + "loss": 1.161, + "theoretical_loss": 3.418607625516038, + "tokens_seen": 2084634624 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037464646464646466, + "loss": 1.7278, + "theoretical_loss": 3.4185987376889866, + "tokens_seen": 2084700160 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037462626262626267, + "loss": 1.3251, + "theoretical_loss": 3.418589850219565, + "tokens_seen": 2084765696 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037460606060606063, + "loss": 1.4643, + "theoretical_loss": 3.418580963107746, + "tokens_seen": 2084831232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745858585858586, + "loss": 1.6184, + "theoretical_loss": 3.418572076353506, + "tokens_seen": 2084896768 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037456565656565654, + "loss": 1.1971, + "theoretical_loss": 3.4185631899568185, + "tokens_seen": 2084962304 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037454545454545455, + "loss": 1.5665, + "theoretical_loss": 3.4185543039176576, + "tokens_seen": 2085027840 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037452525252525256, + "loss": 1.2791, + "theoretical_loss": 3.4185454182359982, + "tokens_seen": 2085093376 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745050505050505, + "loss": 1.4199, + "theoretical_loss": 3.418536532911814, + "tokens_seen": 2085158912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744848484848485, + "loss": 1.3007, + "theoretical_loss": 3.41852764794508, + "tokens_seen": 2085224448 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037446464646464643, + "loss": 1.2452, + "theoretical_loss": 3.4185187633357703, + "tokens_seen": 2085289984 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744444444444445, + "loss": 1.4831, + "theoretical_loss": 3.4185098790838593, + "tokens_seen": 2085355520 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037442424242424246, + "loss": 1.4045, + "theoretical_loss": 3.418500995189322, + "tokens_seen": 2085421056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744040404040404, + "loss": 1.6719, + "theoretical_loss": 3.4184921116521316, + "tokens_seen": 2085486592 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037438383838383837, + "loss": 1.3131, + "theoretical_loss": 3.4184832284722635, + "tokens_seen": 2085552128 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743636363636363, + "loss": 1.4928, + "theoretical_loss": 3.4184743456496918, + "tokens_seen": 2085617664 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743434343434344, + "loss": 1.5779, + "theoretical_loss": 3.4184654631843907, + "tokens_seen": 2085683200 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037432323232323235, + "loss": 1.4453, + "theoretical_loss": 3.4184565810763345, + "tokens_seen": 2085748736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743030303030303, + "loss": 1.208, + "theoretical_loss": 3.4184476993254984, + "tokens_seen": 2085814272 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037428282828282826, + "loss": 1.4881, + "theoretical_loss": 3.4184388179318557, + "tokens_seen": 2085879808 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.3360215127468109, + "objective/train/avg_token_score": 0.022043464705348015, + "objective/train/docs_used": 1179467, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1368883848190308, + "objective/train/original_loss": 4.677212238311768, + "objective/train/theoretical_loss": 3.4184299368953814, + "objective/train/tokens_used": 444804576, + "objective/train/unlikelihood_loss": 0.06505218893289566, + "theoretical_loss": 3.4184299368953814, + "tokens_seen": 2085945344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742626262626263, + "loss": 1.338, + "theoretical_loss": 3.4184299368953814, + "tokens_seen": 2085945344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742424242424243, + "loss": 1.3168, + "theoretical_loss": 3.4184210562160504, + "tokens_seen": 2086010880 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037422222222222224, + "loss": 1.5364, + "theoretical_loss": 3.4184121758938364, + "tokens_seen": 2086076416 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742020202020202, + "loss": 1.4987, + "theoretical_loss": 3.418403295928714, + "tokens_seen": 2086141952 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037418181818181815, + "loss": 1.2955, + "theoretical_loss": 3.4183944163206577, + "tokens_seen": 2086207488 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037416161616161617, + "loss": 1.5719, + "theoretical_loss": 3.418385537069642, + "tokens_seen": 2086273024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741414141414142, + "loss": 1.562, + "theoretical_loss": 3.4183766581756405, + "tokens_seen": 2086338560 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037412121212121213, + "loss": 1.2294, + "theoretical_loss": 3.418367779638629, + "tokens_seen": 2086404096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741010101010101, + "loss": 1.5268, + "theoretical_loss": 3.418358901458581, + "tokens_seen": 2086469632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740808080808081, + "loss": 1.4696, + "theoretical_loss": 3.4183500236354716, + "tokens_seen": 2086535168 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037406060606060606, + "loss": 1.4381, + "theoretical_loss": 3.4183411461692748, + "tokens_seen": 2086600704 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037404040404040407, + "loss": 1.3861, + "theoretical_loss": 3.418332269059965, + "tokens_seen": 2086666240 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374020202020202, + "loss": 1.5148, + "theoretical_loss": 3.4183233923075163, + "tokens_seen": 2086731776 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374, + "loss": 1.4125, + "theoretical_loss": 3.418314515911904, + "tokens_seen": 2086797312 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373979797979798, + "loss": 1.3945, + "theoretical_loss": 3.418305639873102, + "tokens_seen": 2086862848 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037395959595959595, + "loss": 1.4482, + "theoretical_loss": 3.4182967641910853, + "tokens_seen": 2086928384 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037393939393939396, + "loss": 1.4271, + "theoretical_loss": 3.4182878888658275, + "tokens_seen": 2086993920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739191919191919, + "loss": 1.3665, + "theoretical_loss": 3.418279013897304, + "tokens_seen": 2087059456 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037389898989898993, + "loss": 1.3543, + "theoretical_loss": 3.418270139285488, + "tokens_seen": 2087124992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738787878787879, + "loss": 1.5821, + "theoretical_loss": 3.418261265030355, + "tokens_seen": 2087190528 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037385858585858584, + "loss": 1.671, + "theoretical_loss": 3.4182523911318796, + "tokens_seen": 2087256064 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037383838383838386, + "loss": 1.2803, + "theoretical_loss": 3.418243517590036, + "tokens_seen": 2087321600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738181818181818, + "loss": 1.4464, + "theoretical_loss": 3.418234644404798, + "tokens_seen": 2087387136 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737979797979798, + "loss": 1.624, + "theoretical_loss": 3.4182257715761404, + "tokens_seen": 2087452672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737777777777778, + "loss": 1.2767, + "theoretical_loss": 3.4182168991040385, + "tokens_seen": 2087518208 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.3165322542190552, + "objective/train/avg_token_score": 0.0110182398930192, + "objective/train/docs_used": 1180869, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0291162729263306, + "objective/train/original_loss": 4.169273853302002, + "objective/train/theoretical_loss": 3.4182080269884656, + "objective/train/tokens_used": 446442976, + "objective/train/unlikelihood_loss": 0.08053578436374664, + "theoretical_loss": 3.4182080269884656, + "tokens_seen": 2087583744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037375757575757574, + "loss": 1.2945, + "theoretical_loss": 3.4182080269884656, + "tokens_seen": 2087583744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037373737373737375, + "loss": 1.4622, + "theoretical_loss": 3.4181991552293973, + "tokens_seen": 2087649280 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037371717171717176, + "loss": 1.4116, + "theoretical_loss": 3.4181902838268075, + "tokens_seen": 2087714816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736969696969697, + "loss": 1.4191, + "theoretical_loss": 3.4181814127806702, + "tokens_seen": 2087780352 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037367676767676767, + "loss": 1.4904, + "theoretical_loss": 3.4181725420909608, + "tokens_seen": 2087845888 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037365656565656563, + "loss": 1.3255, + "theoretical_loss": 3.4181636717576533, + "tokens_seen": 2087911424 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037363636363636364, + "loss": 1.4168, + "theoretical_loss": 3.418154801780722, + "tokens_seen": 2087976960 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037361616161616165, + "loss": 1.6034, + "theoretical_loss": 3.418145932160142, + "tokens_seen": 2088042496 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735959595959596, + "loss": 1.3212, + "theoretical_loss": 3.4181370628958874, + "tokens_seen": 2088108032 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037357575757575757, + "loss": 1.437, + "theoretical_loss": 3.4181281939879327, + "tokens_seen": 2088173568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735555555555555, + "loss": 1.3466, + "theoretical_loss": 3.418119325436253, + "tokens_seen": 2088239104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735353535353536, + "loss": 1.4281, + "theoretical_loss": 3.4181104572408216, + "tokens_seen": 2088304640 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037351515151515154, + "loss": 1.6064, + "theoretical_loss": 3.4181015894016142, + "tokens_seen": 2088370176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734949494949495, + "loss": 1.2708, + "theoretical_loss": 3.4180927219186046, + "tokens_seen": 2088435712 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037347474747474746, + "loss": 1.195, + "theoretical_loss": 3.418083854791768, + "tokens_seen": 2088501248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734545454545454, + "loss": 1.4323, + "theoretical_loss": 3.4180749880210777, + "tokens_seen": 2088566784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734343434343435, + "loss": 1.5022, + "theoretical_loss": 3.4180661216065094, + "tokens_seen": 2088632320 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037341414141414144, + "loss": 1.299, + "theoretical_loss": 3.418057255548037, + "tokens_seen": 2088697856 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733939393939394, + "loss": 1.3051, + "theoretical_loss": 3.4180483898456355, + "tokens_seen": 2088763392 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037337373737373735, + "loss": 1.7103, + "theoretical_loss": 3.418039524499279, + "tokens_seen": 2088828928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733535353535353, + "loss": 1.4726, + "theoretical_loss": 3.418030659508942, + "tokens_seen": 2088894464 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733333333333334, + "loss": 1.8611, + "theoretical_loss": 3.4180217948745995, + "tokens_seen": 2088960000 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037331313131313133, + "loss": 1.5577, + "theoretical_loss": 3.418012930596226, + "tokens_seen": 2089025536 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732929292929293, + "loss": 1.7129, + "theoretical_loss": 3.418004066673795, + "tokens_seen": 2089091072 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037327272727272724, + "loss": 1.6059, + "theoretical_loss": 3.4179952031072824, + "tokens_seen": 2089156608 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.4984726309776306, + "objective/train/avg_token_score": 0.008942843414843082, + "objective/train/docs_used": 1181528, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7859277725219727, + "objective/train/original_loss": 4.169965744018555, + "objective/train/theoretical_loss": 3.417986339896662, + "objective/train/tokens_used": 448081376, + "objective/train/unlikelihood_loss": 0.05391286313533783, + "theoretical_loss": 3.417986339896662, + "tokens_seen": 2089222144 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732525252525253, + "loss": 1.5706, + "theoretical_loss": 3.417986339896662, + "tokens_seen": 2089222144 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037323232323232327, + "loss": 1.4367, + "theoretical_loss": 3.4179774770419087, + "tokens_seen": 2089287680 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732121212121212, + "loss": 1.4038, + "theoretical_loss": 3.417968614542997, + "tokens_seen": 2089353216 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731919191919192, + "loss": 1.4713, + "theoretical_loss": 3.4179597523999012, + "tokens_seen": 2089418752 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037317171717171714, + "loss": 1.5628, + "theoretical_loss": 3.4179508906125955, + "tokens_seen": 2089484288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731515151515152, + "loss": 1.5289, + "theoretical_loss": 3.417942029181056, + "tokens_seen": 2089549824 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037313131313131316, + "loss": 1.481, + "theoretical_loss": 3.417933168105255, + "tokens_seen": 2089615360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731111111111111, + "loss": 1.6574, + "theoretical_loss": 3.417924307385169, + "tokens_seen": 2089680896 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037309090909090907, + "loss": 1.201, + "theoretical_loss": 3.4179154470207718, + "tokens_seen": 2089746432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730707070707071, + "loss": 1.2907, + "theoretical_loss": 3.4179065870120375, + "tokens_seen": 2089811968 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730505050505051, + "loss": 1.3189, + "theoretical_loss": 3.417897727358942, + "tokens_seen": 2089877504 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037303030303030305, + "loss": 1.6222, + "theoretical_loss": 3.4178888680614583, + "tokens_seen": 2089943040 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373010101010101, + "loss": 1.4959, + "theoretical_loss": 3.4178800091195622, + "tokens_seen": 2090008576 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037298989898989896, + "loss": 1.2521, + "theoretical_loss": 3.4178711505332275, + "tokens_seen": 2090074112 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372969696969697, + "loss": 1.6979, + "theoretical_loss": 3.417862292302429, + "tokens_seen": 2090139648 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372949494949495, + "loss": 1.6717, + "theoretical_loss": 3.4178534344271414, + "tokens_seen": 2090205184 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037292929292929294, + "loss": 1.4221, + "theoretical_loss": 3.4178445769073393, + "tokens_seen": 2090270720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729090909090909, + "loss": 1.424, + "theoretical_loss": 3.417835719742998, + "tokens_seen": 2090336256 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728888888888889, + "loss": 1.5732, + "theoretical_loss": 3.41782686293409, + "tokens_seen": 2090401792 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037286868686868687, + "loss": 1.4647, + "theoretical_loss": 3.417818006480592, + "tokens_seen": 2090467328 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728484848484849, + "loss": 1.6327, + "theoretical_loss": 3.4178091503824777, + "tokens_seen": 2090532864 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037282828282828284, + "loss": 1.4232, + "theoretical_loss": 3.4178002946397217, + "tokens_seen": 2090598400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728080808080808, + "loss": 1.383, + "theoretical_loss": 3.417791439252299, + "tokens_seen": 2090663936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727878787878788, + "loss": 1.3174, + "theoretical_loss": 3.4177825842201837, + "tokens_seen": 2090729472 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037276767676767676, + "loss": 1.4924, + "theoretical_loss": 3.4177737295433506, + "tokens_seen": 2090795008 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.47623410820961, + "objective/train/avg_token_score": 0.006108488887548447, + "objective/train/docs_used": 1182667, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4925786256790161, + "objective/train/original_loss": 3.958449125289917, + "objective/train/theoretical_loss": 3.417764875221774, + "objective/train/tokens_used": 449719776, + "objective/train/unlikelihood_loss": 0.08952859044075012, + "theoretical_loss": 3.417764875221774, + "tokens_seen": 2090860544 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037274747474747477, + "loss": 1.5307, + "theoretical_loss": 3.417764875221774, + "tokens_seen": 2090860544 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037272727272727273, + "loss": 1.456, + "theoretical_loss": 3.4177560212554297, + "tokens_seen": 2090926080 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037270707070707074, + "loss": 1.3673, + "theoretical_loss": 3.4177471676442908, + "tokens_seen": 2090991616 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726868686868687, + "loss": 1.5375, + "theoretical_loss": 3.417738314388333, + "tokens_seen": 2091057152 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037266666666666665, + "loss": 1.188, + "theoretical_loss": 3.4177294614875304, + "tokens_seen": 2091122688 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037264646464646466, + "loss": 1.4103, + "theoretical_loss": 3.4177206089418575, + "tokens_seen": 2091188224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726262626262626, + "loss": 1.5421, + "theoretical_loss": 3.4177117567512894, + "tokens_seen": 2091253760 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037260606060606063, + "loss": 1.17, + "theoretical_loss": 3.4177029049158003, + "tokens_seen": 2091319296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725858585858586, + "loss": 1.6131, + "theoretical_loss": 3.4176940534353655, + "tokens_seen": 2091384832 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037256565656565655, + "loss": 1.4615, + "theoretical_loss": 3.417685202309958, + "tokens_seen": 2091450368 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037254545454545456, + "loss": 1.4895, + "theoretical_loss": 3.417676351539555, + "tokens_seen": 2091515904 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037252525252525257, + "loss": 1.3297, + "theoretical_loss": 3.4176675011241286, + "tokens_seen": 2091581440 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725050505050505, + "loss": 1.2792, + "theoretical_loss": 3.417658651063655, + "tokens_seen": 2091646976 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724848484848485, + "loss": 1.5627, + "theoretical_loss": 3.4176498013581087, + "tokens_seen": 2091712512 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037246464646464644, + "loss": 1.3112, + "theoretical_loss": 3.4176409520074635, + "tokens_seen": 2091778048 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037244444444444445, + "loss": 1.3046, + "theoretical_loss": 3.417632103011695, + "tokens_seen": 2091843584 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037242424242424246, + "loss": 1.5259, + "theoretical_loss": 3.417623254370777, + "tokens_seen": 2091909120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724040404040404, + "loss": 1.4383, + "theoretical_loss": 3.4176144060846854, + "tokens_seen": 2091974656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723838383838384, + "loss": 1.2748, + "theoretical_loss": 3.4176055581533933, + "tokens_seen": 2092040192 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037236363636363633, + "loss": 1.6042, + "theoretical_loss": 3.417596710576876, + "tokens_seen": 2092105728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723434343434344, + "loss": 1.3628, + "theoretical_loss": 3.417587863355109, + "tokens_seen": 2092171264 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037232323232323235, + "loss": 1.4676, + "theoretical_loss": 3.417579016488066, + "tokens_seen": 2092236800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723030303030303, + "loss": 1.334, + "theoretical_loss": 3.417570169975722, + "tokens_seen": 2092302336 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037228282828282827, + "loss": 1.3142, + "theoretical_loss": 3.417561323818051, + "tokens_seen": 2092367872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722626262626262, + "loss": 1.3255, + "theoretical_loss": 3.417552478015029, + "tokens_seen": 2092433408 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.4452590346336365, + "objective/train/avg_token_score": 0.009709584526717663, + "objective/train/docs_used": 1183291, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5259029865264893, + "objective/train/original_loss": 4.389416694641113, + "objective/train/theoretical_loss": 3.4175436325666295, + "objective/train/tokens_used": 451358176, + "objective/train/unlikelihood_loss": 0.06972787529230118, + "theoretical_loss": 3.4175436325666295, + "tokens_seen": 2092498944 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722424242424243, + "loss": 1.3672, + "theoretical_loss": 3.4175436325666295, + "tokens_seen": 2092498944 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037222222222222225, + "loss": 1.3865, + "theoretical_loss": 3.4175347874728277, + "tokens_seen": 2092564480 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722020202020202, + "loss": 1.6353, + "theoretical_loss": 3.417525942733598, + "tokens_seen": 2092630016 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037218181818181816, + "loss": 1.3421, + "theoretical_loss": 3.417517098348916, + "tokens_seen": 2092695552 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037216161616161617, + "loss": 1.3906, + "theoretical_loss": 3.4175082543187547, + "tokens_seen": 2092761088 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721414141414142, + "loss": 1.4974, + "theoretical_loss": 3.4174994106430905, + "tokens_seen": 2092826624 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037212121212121214, + "loss": 1.567, + "theoretical_loss": 3.417490567321897, + "tokens_seen": 2092892160 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721010101010101, + "loss": 1.5231, + "theoretical_loss": 3.4174817243551487, + "tokens_seen": 2092957696 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037208080808080805, + "loss": 1.5456, + "theoretical_loss": 3.4174728817428215, + "tokens_seen": 2093023232 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037206060606060606, + "loss": 1.4819, + "theoretical_loss": 3.417464039484889, + "tokens_seen": 2093088768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720404040404041, + "loss": 1.5072, + "theoretical_loss": 3.4174551975813268, + "tokens_seen": 2093154304 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037202020202020203, + "loss": 1.5806, + "theoretical_loss": 3.417446356032109, + "tokens_seen": 2093219840 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372, + "loss": 1.5619, + "theoretical_loss": 3.4174375148372103, + "tokens_seen": 2093285376 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371979797979798, + "loss": 1.5147, + "theoretical_loss": 3.4174286739966053, + "tokens_seen": 2093350912 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037195959595959596, + "loss": 1.3901, + "theoretical_loss": 3.4174198335102695, + "tokens_seen": 2093416448 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037193939393939397, + "loss": 1.3022, + "theoretical_loss": 3.417410993378177, + "tokens_seen": 2093481984 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003719191919191919, + "loss": 1.6844, + "theoretical_loss": 3.4174021536003023, + "tokens_seen": 2093547520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718989898989899, + "loss": 1.6104, + "theoretical_loss": 3.4173933141766204, + "tokens_seen": 2093613056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718787878787879, + "loss": 1.4473, + "theoretical_loss": 3.417384475107106, + "tokens_seen": 2093678592 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037185858585858585, + "loss": 1.3699, + "theoretical_loss": 3.417375636391734, + "tokens_seen": 2093744128 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037183838383838386, + "loss": 1.5994, + "theoretical_loss": 3.417366798030479, + "tokens_seen": 2093809664 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718181818181818, + "loss": 1.3916, + "theoretical_loss": 3.4173579600233155, + "tokens_seen": 2093875200 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037179797979797983, + "loss": 1.2843, + "theoretical_loss": 3.4173491223702186, + "tokens_seen": 2093940736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717777777777778, + "loss": 1.5654, + "theoretical_loss": 3.417340285071163, + "tokens_seen": 2094006272 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037175757575757574, + "loss": 1.6744, + "theoretical_loss": 3.4173314481261228, + "tokens_seen": 2094071808 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.4912634491920471, + "objective/train/avg_token_score": 0.00644297618418932, + "objective/train/docs_used": 1183994, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7411361932754517, + "objective/train/original_loss": 4.177827835083008, + "objective/train/theoretical_loss": 3.4173226115350737, + "objective/train/tokens_used": 452996576, + "objective/train/unlikelihood_loss": 0.0659017562866211, + "theoretical_loss": 3.4173226115350737, + "tokens_seen": 2094137344 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037173737373737375, + "loss": 1.5447, + "theoretical_loss": 3.4173226115350737, + "tokens_seen": 2094137344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717171717171717, + "loss": 1.4195, + "theoretical_loss": 3.41731377529799, + "tokens_seen": 2094202880 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716969696969697, + "loss": 1.4795, + "theoretical_loss": 3.417304939414846, + "tokens_seen": 2094268416 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716767676767677, + "loss": 1.3937, + "theoretical_loss": 3.4172961038856173, + "tokens_seen": 2094333952 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037165656565656564, + "loss": 1.2934, + "theoretical_loss": 3.417287268710278, + "tokens_seen": 2094399488 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037163636363636365, + "loss": 1.4347, + "theoretical_loss": 3.4172784338888036, + "tokens_seen": 2094465024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716161616161616, + "loss": 1.357, + "theoretical_loss": 3.417269599421168, + "tokens_seen": 2094530560 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715959595959596, + "loss": 1.3482, + "theoretical_loss": 3.417260765307346, + "tokens_seen": 2094596096 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037157575757575757, + "loss": 1.4318, + "theoretical_loss": 3.417251931547313, + "tokens_seen": 2094661632 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037155555555555553, + "loss": 1.3884, + "theoretical_loss": 3.417243098141043, + "tokens_seen": 2094727168 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037153535353535354, + "loss": 1.3685, + "theoretical_loss": 3.4172342650885117, + "tokens_seen": 2094792704 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037151515151515155, + "loss": 1.4191, + "theoretical_loss": 3.417225432389693, + "tokens_seen": 2094858240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714949494949495, + "loss": 1.2099, + "theoretical_loss": 3.4172166000445623, + "tokens_seen": 2094923776 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037147474747474746, + "loss": 1.491, + "theoretical_loss": 3.417207768053094, + "tokens_seen": 2094989312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714545454545455, + "loss": 1.4384, + "theoretical_loss": 3.4171989364152626, + "tokens_seen": 2095054848 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037143434343434343, + "loss": 1.6164, + "theoretical_loss": 3.417190105131044, + "tokens_seen": 2095120384 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037141414141414144, + "loss": 1.4524, + "theoretical_loss": 3.4171812742004115, + "tokens_seen": 2095185920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713939393939394, + "loss": 1.3992, + "theoretical_loss": 3.417172443623341, + "tokens_seen": 2095251456 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037137373737373736, + "loss": 1.3847, + "theoretical_loss": 3.417163613399807, + "tokens_seen": 2095316992 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037135353535353537, + "loss": 1.6524, + "theoretical_loss": 3.417154783529784, + "tokens_seen": 2095382528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713333333333334, + "loss": 1.468, + "theoretical_loss": 3.4171459540132467, + "tokens_seen": 2095448064 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037131313131313134, + "loss": 1.2954, + "theoretical_loss": 3.4171371248501705, + "tokens_seen": 2095513600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712929292929293, + "loss": 1.559, + "theoretical_loss": 3.41712829604053, + "tokens_seen": 2095579136 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037127272727272725, + "loss": 1.4466, + "theoretical_loss": 3.4171194675842997, + "tokens_seen": 2095644672 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037125252525252526, + "loss": 1.4198, + "theoretical_loss": 3.4171106394814545, + "tokens_seen": 2095710208 + }, + { + "epoch": 0.26, + "objective/train/aligned_token_count": 0.28158602118492126, + "objective/train/avg_token_score": 0.014217756688594818, + "objective/train/docs_used": 1184873, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8266394734382629, + "objective/train/original_loss": 5.028609275817871, + "objective/train/theoretical_loss": 3.41710181173197, + "objective/train/tokens_used": 454634976, + "objective/train/unlikelihood_loss": 0.05122791603207588, + "theoretical_loss": 3.41710181173197, + "tokens_seen": 2095775744 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037123232323232327, + "loss": 1.256, + "theoretical_loss": 3.41710181173197, + "tokens_seen": 2095775744 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037121212121212123, + "loss": 1.3978, + "theoretical_loss": 3.4170929843358193, + "tokens_seen": 2095841280 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711919191919192, + "loss": 1.3955, + "theoretical_loss": 3.4170841572929787, + "tokens_seen": 2095906816 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037117171717171714, + "loss": 1.3538, + "theoretical_loss": 3.4170753306034225, + "tokens_seen": 2095972352 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711515151515152, + "loss": 1.2636, + "theoretical_loss": 3.417066504267125, + "tokens_seen": 2096037888 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037113131313131316, + "loss": 1.4575, + "theoretical_loss": 3.4170576782840625, + "tokens_seen": 2096103424 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711111111111111, + "loss": 1.4449, + "theoretical_loss": 3.417048852654208, + "tokens_seen": 2096168960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003710909090909091, + "loss": 1.2891, + "theoretical_loss": 3.4170400273775376, + "tokens_seen": 2096234496 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037107070707070703, + "loss": 1.4142, + "theoretical_loss": 3.4170312024540257, + "tokens_seen": 2096300032 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003710505050505051, + "loss": 1.4076, + "theoretical_loss": 3.4170223778836473, + "tokens_seen": 2096365568 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037103030303030306, + "loss": 1.2815, + "theoretical_loss": 3.4170135536663766, + "tokens_seen": 2096431104 + }, + { + "epoch": 0.27, + "learning_rate": 0.000371010101010101, + "loss": 1.3764, + "theoretical_loss": 3.4170047298021893, + "tokens_seen": 2096496640 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037098989898989897, + "loss": 1.6844, + "theoretical_loss": 3.41699590629106, + "tokens_seen": 2096562176 + }, + { + "epoch": 0.27, + "learning_rate": 0.000370969696969697, + "loss": 1.626, + "theoretical_loss": 3.4169870831329625, + "tokens_seen": 2096627712 + }, + { + "epoch": 0.27, + "learning_rate": 0.000370949494949495, + "loss": 1.419, + "theoretical_loss": 3.4169782603278733, + "tokens_seen": 2096693248 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037092929292929295, + "loss": 1.4495, + "theoretical_loss": 3.4169694378757662, + "tokens_seen": 2096758784 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709090909090909, + "loss": 1.3304, + "theoretical_loss": 3.4169606157766164, + "tokens_seen": 2096824320 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037088888888888886, + "loss": 1.3579, + "theoretical_loss": 3.416951794030399, + "tokens_seen": 2096889856 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708686868686869, + "loss": 1.4558, + "theoretical_loss": 3.416942972637088, + "tokens_seen": 2096955392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708484848484849, + "loss": 1.2601, + "theoretical_loss": 3.4169341515966583, + "tokens_seen": 2097020928 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037082828282828284, + "loss": 1.4484, + "theoretical_loss": 3.416925330909086, + "tokens_seen": 2097086464 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708080808080808, + "loss": 1.393, + "theoretical_loss": 3.416916510574345, + "tokens_seen": 2097152000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707878787878788, + "loss": 1.3275, + "theoretical_loss": 3.41690769059241, + "tokens_seen": 2097217536 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037076767676767677, + "loss": 1.4141, + "theoretical_loss": 3.416898870963257, + "tokens_seen": 2097283072 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707474747474748, + "loss": 1.5404, + "theoretical_loss": 3.416890051686859, + "tokens_seen": 2097348608 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.5108137726783752, + "objective/train/avg_token_score": 0.02312566339969635, + "objective/train/docs_used": 1185597, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7117425203323364, + "objective/train/original_loss": 4.583292484283447, + "objective/train/theoretical_loss": 3.4168812327631928, + "objective/train/tokens_used": 456273376, + "objective/train/unlikelihood_loss": 0.07427631318569183, + "theoretical_loss": 3.4168812327631928, + "tokens_seen": 2097414144 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037072727272727273, + "loss": 1.4591, + "theoretical_loss": 3.4168812327631928, + "tokens_seen": 2097414144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707070707070707, + "loss": 1.4745, + "theoretical_loss": 3.416872414192232, + "tokens_seen": 2097479680 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706868686868687, + "loss": 1.4241, + "theoretical_loss": 3.416863595973952, + "tokens_seen": 2097545216 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037066666666666666, + "loss": 1.3937, + "theoretical_loss": 3.4168547781083274, + "tokens_seen": 2097610752 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037064646464646467, + "loss": 1.6599, + "theoretical_loss": 3.4168459605953334, + "tokens_seen": 2097676288 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037062626262626263, + "loss": 1.4604, + "theoretical_loss": 3.416837143434945, + "tokens_seen": 2097741824 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037060606060606064, + "loss": 1.3529, + "theoretical_loss": 3.416828326627136, + "tokens_seen": 2097807360 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003705858585858586, + "loss": 1.3973, + "theoretical_loss": 3.416819510171883, + "tokens_seen": 2097872896 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037056565656565655, + "loss": 1.3854, + "theoretical_loss": 3.4168106940691594, + "tokens_seen": 2097938432 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037054545454545456, + "loss": 1.3052, + "theoretical_loss": 3.4168018783189407, + "tokens_seen": 2098003968 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003705252525252525, + "loss": 1.2121, + "theoretical_loss": 3.4167930629212018, + "tokens_seen": 2098069504 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037050505050505053, + "loss": 1.3941, + "theoretical_loss": 3.416784247875918, + "tokens_seen": 2098135040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704848484848485, + "loss": 1.5037, + "theoretical_loss": 3.416775433183063, + "tokens_seen": 2098200576 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037046464646464644, + "loss": 1.4388, + "theoretical_loss": 3.4167666188426127, + "tokens_seen": 2098266112 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037044444444444446, + "loss": 1.4765, + "theoretical_loss": 3.4167578048545417, + "tokens_seen": 2098331648 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037042424242424247, + "loss": 1.5656, + "theoretical_loss": 3.4167489912188254, + "tokens_seen": 2098397184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704040404040404, + "loss": 1.3592, + "theoretical_loss": 3.4167401779354383, + "tokens_seen": 2098462720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703838383838384, + "loss": 1.6362, + "theoretical_loss": 3.416731365004355, + "tokens_seen": 2098528256 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037036363636363634, + "loss": 1.5223, + "theoretical_loss": 3.4167225524255507, + "tokens_seen": 2098593792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037034343434343435, + "loss": 1.5873, + "theoretical_loss": 3.4167137401990004, + "tokens_seen": 2098659328 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037032323232323236, + "loss": 1.3729, + "theoretical_loss": 3.4167049283246786, + "tokens_seen": 2098724864 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703030303030303, + "loss": 1.3783, + "theoretical_loss": 3.4166961168025614, + "tokens_seen": 2098790400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702828282828283, + "loss": 1.4138, + "theoretical_loss": 3.4166873056326224, + "tokens_seen": 2098855936 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037026262626262623, + "loss": 1.4193, + "theoretical_loss": 3.416678494814837, + "tokens_seen": 2098921472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702424242424243, + "loss": 1.5558, + "theoretical_loss": 3.4166696843491806, + "tokens_seen": 2098987008 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.3702346086502075, + "objective/train/avg_token_score": 0.02101566642522812, + "objective/train/docs_used": 1186813, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3314353227615356, + "objective/train/original_loss": 4.864157199859619, + "objective/train/theoretical_loss": 3.4166608742356273, + "objective/train/tokens_used": 457911776, + "objective/train/unlikelihood_loss": 0.05566110089421272, + "theoretical_loss": 3.4166608742356273, + "tokens_seen": 2099052544 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037022222222222225, + "loss": 1.2552, + "theoretical_loss": 3.4166608742356273, + "tokens_seen": 2099052544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702020202020202, + "loss": 1.4825, + "theoretical_loss": 3.4166520644741523, + "tokens_seen": 2099118080 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037018181818181817, + "loss": 1.3166, + "theoretical_loss": 3.4166432550647308, + "tokens_seen": 2099183616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003701616161616161, + "loss": 1.3974, + "theoretical_loss": 3.4166344460073375, + "tokens_seen": 2099249152 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003701414141414142, + "loss": 1.7226, + "theoretical_loss": 3.416625637301948, + "tokens_seen": 2099314688 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037012121212121215, + "loss": 1.3935, + "theoretical_loss": 3.416616828948536, + "tokens_seen": 2099380224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003701010101010101, + "loss": 1.3225, + "theoretical_loss": 3.4166080209470775, + "tokens_seen": 2099445760 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037008080808080806, + "loss": 1.3545, + "theoretical_loss": 3.416599213297547, + "tokens_seen": 2099511296 + }, + { + "epoch": 0.27, + "learning_rate": 0.000370060606060606, + "loss": 1.1832, + "theoretical_loss": 3.4165904059999193, + "tokens_seen": 2099576832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003700404040404041, + "loss": 1.3616, + "theoretical_loss": 3.41658159905417, + "tokens_seen": 2099642368 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037002020202020204, + "loss": 1.326, + "theoretical_loss": 3.416572792460274, + "tokens_seen": 2099707904 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037, + "loss": 1.2092, + "theoretical_loss": 3.4165639862182053, + "tokens_seen": 2099773440 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036997979797979795, + "loss": 1.2938, + "theoretical_loss": 3.4165551803279395, + "tokens_seen": 2099838976 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036995959595959596, + "loss": 1.3802, + "theoretical_loss": 3.416546374789452, + "tokens_seen": 2099904512 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369939393939394, + "loss": 1.4445, + "theoretical_loss": 3.416537569602717, + "tokens_seen": 2099970048 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036991919191919193, + "loss": 1.4575, + "theoretical_loss": 3.41652876476771, + "tokens_seen": 2100035584 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698989898989899, + "loss": 1.2907, + "theoretical_loss": 3.4165199602844054, + "tokens_seen": 2100101120 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036987878787878784, + "loss": 1.5485, + "theoretical_loss": 3.416511156152779, + "tokens_seen": 2100166656 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036985858585858586, + "loss": 1.3936, + "theoretical_loss": 3.416502352372805, + "tokens_seen": 2100232192 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036983838383838387, + "loss": 1.5031, + "theoretical_loss": 3.416493548944459, + "tokens_seen": 2100297728 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698181818181818, + "loss": 1.4496, + "theoretical_loss": 3.4164847458677157, + "tokens_seen": 2100363264 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697979797979798, + "loss": 1.4239, + "theoretical_loss": 3.4164759431425495, + "tokens_seen": 2100428800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697777777777778, + "loss": 1.5725, + "theoretical_loss": 3.4164671407689364, + "tokens_seen": 2100494336 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697575757575758, + "loss": 1.5798, + "theoretical_loss": 3.4164583387468515, + "tokens_seen": 2100559872 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036973737373737376, + "loss": 1.3557, + "theoretical_loss": 3.4164495370762684, + "tokens_seen": 2100625408 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.3393206298351288, + "objective/train/avg_token_score": 0.0204568300396204, + "objective/train/docs_used": 1187470, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.139434814453125, + "objective/train/original_loss": 4.641504287719727, + "objective/train/theoretical_loss": 3.4164407357571633, + "objective/train/tokens_used": 459550176, + "objective/train/unlikelihood_loss": 0.058319222182035446, + "theoretical_loss": 3.4164407357571633, + "tokens_seen": 2100690944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697171717171717, + "loss": 1.3129, + "theoretical_loss": 3.4164407357571633, + "tokens_seen": 2100690944 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036969696969696967, + "loss": 1.4859, + "theoretical_loss": 3.416431934789511, + "tokens_seen": 2100756480 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696767676767677, + "loss": 1.4623, + "theoretical_loss": 3.416423134173286, + "tokens_seen": 2100822016 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696565656565657, + "loss": 1.4411, + "theoretical_loss": 3.416414333908464, + "tokens_seen": 2100887552 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036963636363636365, + "loss": 1.2275, + "theoretical_loss": 3.4164055339950195, + "tokens_seen": 2100953088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696161616161616, + "loss": 1.452, + "theoretical_loss": 3.4163967344329276, + "tokens_seen": 2101018624 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695959595959596, + "loss": 1.5592, + "theoretical_loss": 3.416387935222164, + "tokens_seen": 2101084160 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695757575757576, + "loss": 1.3659, + "theoretical_loss": 3.4163791363627025, + "tokens_seen": 2101149696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695555555555556, + "loss": 1.5916, + "theoretical_loss": 3.4163703378545187, + "tokens_seen": 2101215232 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036953535353535354, + "loss": 1.2753, + "theoretical_loss": 3.4163615396975877, + "tokens_seen": 2101280768 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695151515151515, + "loss": 1.4703, + "theoretical_loss": 3.4163527418918846, + "tokens_seen": 2101346304 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694949494949495, + "loss": 1.3113, + "theoretical_loss": 3.4163439444373846, + "tokens_seen": 2101411840 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036947474747474747, + "loss": 1.4597, + "theoretical_loss": 3.416335147334062, + "tokens_seen": 2101477376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694545454545455, + "loss": 1.4478, + "theoretical_loss": 3.416326350581892, + "tokens_seen": 2101542912 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036943434343434344, + "loss": 1.4999, + "theoretical_loss": 3.416317554180851, + "tokens_seen": 2101608448 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036941414141414145, + "loss": 1.5301, + "theoretical_loss": 3.4163087581309117, + "tokens_seen": 2101673984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693939393939394, + "loss": 1.404, + "theoretical_loss": 3.416299962432051, + "tokens_seen": 2101739520 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036937373737373736, + "loss": 1.3892, + "theoretical_loss": 3.4162911670842435, + "tokens_seen": 2101805056 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693535353535354, + "loss": 1.569, + "theoretical_loss": 3.4162823720874633, + "tokens_seen": 2101870592 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036933333333333333, + "loss": 1.7534, + "theoretical_loss": 3.416273577441687, + "tokens_seen": 2101936128 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036931313131313134, + "loss": 1.5531, + "theoretical_loss": 3.416264783146888, + "tokens_seen": 2102001664 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692929292929293, + "loss": 1.6067, + "theoretical_loss": 3.4162559892030426, + "tokens_seen": 2102067200 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036927272727272725, + "loss": 1.3843, + "theoretical_loss": 3.4162471956101257, + "tokens_seen": 2102132736 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036925252525252527, + "loss": 1.2177, + "theoretical_loss": 3.416238402368112, + "tokens_seen": 2102198272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692323232323233, + "loss": 1.6496, + "theoretical_loss": 3.4162296094769764, + "tokens_seen": 2102263808 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.3601539731025696, + "objective/train/avg_token_score": 0.05041568726301193, + "objective/train/docs_used": 1188148, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1241708993911743, + "objective/train/original_loss": 4.18945837020874, + "objective/train/theoretical_loss": 3.4162208169366943, + "objective/train/tokens_used": 461188576, + "objective/train/unlikelihood_loss": 0.07274238765239716, + "theoretical_loss": 3.4162208169366943, + "tokens_seen": 2102329344 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036921212121212123, + "loss": 1.5808, + "theoretical_loss": 3.4162208169366943, + "tokens_seen": 2102329344 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691919191919192, + "loss": 1.5366, + "theoretical_loss": 3.416212024747241, + "tokens_seen": 2102394880 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036917171717171715, + "loss": 1.5601, + "theoretical_loss": 3.416203232908591, + "tokens_seen": 2102460416 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036915151515151516, + "loss": 1.3181, + "theoretical_loss": 3.4161944414207195, + "tokens_seen": 2102525952 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036913131313131317, + "loss": 1.5095, + "theoretical_loss": 3.4161856502836017, + "tokens_seen": 2102591488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691111111111111, + "loss": 1.4926, + "theoretical_loss": 3.416176859497213, + "tokens_seen": 2102657024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690909090909091, + "loss": 1.2238, + "theoretical_loss": 3.416168069061528, + "tokens_seen": 2102722560 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036907070707070704, + "loss": 1.233, + "theoretical_loss": 3.416159278976522, + "tokens_seen": 2102788096 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690505050505051, + "loss": 1.4327, + "theoretical_loss": 3.4161504892421695, + "tokens_seen": 2102853632 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036903030303030306, + "loss": 1.3397, + "theoretical_loss": 3.4161416998584464, + "tokens_seen": 2102919168 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369010101010101, + "loss": 1.5452, + "theoretical_loss": 3.4161329108253278, + "tokens_seen": 2102984704 + }, + { + "epoch": 0.27, + "learning_rate": 0.000368989898989899, + "loss": 1.1889, + "theoretical_loss": 3.4161241221427883, + "tokens_seen": 2103050240 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036896969696969693, + "loss": 1.3946, + "theoretical_loss": 3.416115333810803, + "tokens_seen": 2103115776 + }, + { + "epoch": 0.27, + "learning_rate": 0.000368949494949495, + "loss": 1.517, + "theoretical_loss": 3.416106545829347, + "tokens_seen": 2103181312 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036892929292929295, + "loss": 1.5197, + "theoretical_loss": 3.4160977581983962, + "tokens_seen": 2103246848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003689090909090909, + "loss": 1.2718, + "theoretical_loss": 3.4160889709179245, + "tokens_seen": 2103312384 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036888888888888887, + "loss": 1.408, + "theoretical_loss": 3.4160801839879076, + "tokens_seen": 2103377920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688686868686869, + "loss": 1.4539, + "theoretical_loss": 3.4160713974083206, + "tokens_seen": 2103443456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688484848484849, + "loss": 1.4745, + "theoretical_loss": 3.4160626111791386, + "tokens_seen": 2103508992 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036882828282828285, + "loss": 1.2958, + "theoretical_loss": 3.4160538253003367, + "tokens_seen": 2103574528 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688080808080808, + "loss": 1.1511, + "theoretical_loss": 3.41604503977189, + "tokens_seen": 2103640064 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036878787878787876, + "loss": 1.48, + "theoretical_loss": 3.4160362545937732, + "tokens_seen": 2103705600 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036876767676767677, + "loss": 1.3437, + "theoretical_loss": 3.4160274697659623, + "tokens_seen": 2103771136 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003687474747474748, + "loss": 1.6896, + "theoretical_loss": 3.4160186852884316, + "tokens_seen": 2103836672 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036872727272727274, + "loss": 1.2146, + "theoretical_loss": 3.416009901161157, + "tokens_seen": 2103902208 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.4290078282356262, + "objective/train/avg_token_score": 0.015303338877856731, + "objective/train/docs_used": 1189669, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3741037845611572, + "objective/train/original_loss": 4.558312892913818, + "objective/train/theoretical_loss": 3.4160011173841127, + "objective/train/tokens_used": 462826976, + "objective/train/unlikelihood_loss": 0.0731637179851532, + "theoretical_loss": 3.4160011173841127, + "tokens_seen": 2103967744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003687070707070707, + "loss": 1.5371, + "theoretical_loss": 3.4160011173841127, + "tokens_seen": 2103967744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686868686868687, + "loss": 1.2551, + "theoretical_loss": 3.4159923339572744, + "tokens_seen": 2104033280 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036866666666666667, + "loss": 1.5606, + "theoretical_loss": 3.4159835508806173, + "tokens_seen": 2104098816 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686464646464647, + "loss": 1.3252, + "theoretical_loss": 3.4159747681541166, + "tokens_seen": 2104164352 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036862626262626263, + "loss": 1.4486, + "theoretical_loss": 3.4159659857777465, + "tokens_seen": 2104229888 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686060606060606, + "loss": 1.4635, + "theoretical_loss": 3.415957203751484, + "tokens_seen": 2104295424 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685858585858586, + "loss": 1.5143, + "theoretical_loss": 3.415948422075302, + "tokens_seen": 2104360960 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036856565656565656, + "loss": 1.3146, + "theoretical_loss": 3.415939640749177, + "tokens_seen": 2104426496 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036854545454545457, + "loss": 1.4438, + "theoretical_loss": 3.415930859773084, + "tokens_seen": 2104492032 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685252525252525, + "loss": 1.4399, + "theoretical_loss": 3.415922079146998, + "tokens_seen": 2104557568 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036850505050505054, + "loss": 1.4223, + "theoretical_loss": 3.415913298870894, + "tokens_seen": 2104623104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684848484848485, + "loss": 1.4879, + "theoretical_loss": 3.4159045189447474, + "tokens_seen": 2104688640 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036846464646464645, + "loss": 1.4589, + "theoretical_loss": 3.4158957393685334, + "tokens_seen": 2104754176 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036844444444444446, + "loss": 1.4606, + "theoretical_loss": 3.415886960142227, + "tokens_seen": 2104819712 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684242424242424, + "loss": 1.427, + "theoretical_loss": 3.4158781812658034, + "tokens_seen": 2104885248 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036840404040404043, + "loss": 1.4246, + "theoretical_loss": 3.415869402739238, + "tokens_seen": 2104950784 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683838383838384, + "loss": 1.3288, + "theoretical_loss": 3.415860624562505, + "tokens_seen": 2105016320 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036836363636363634, + "loss": 1.444, + "theoretical_loss": 3.415851846735581, + "tokens_seen": 2105081856 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036834343434343435, + "loss": 1.275, + "theoretical_loss": 3.4158430692584405, + "tokens_seen": 2105147392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683232323232323, + "loss": 1.4014, + "theoretical_loss": 3.4158342921310583, + "tokens_seen": 2105212928 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683030303030303, + "loss": 1.5443, + "theoretical_loss": 3.4158255153534096, + "tokens_seen": 2105278464 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003682828282828283, + "loss": 1.4712, + "theoretical_loss": 3.4158167389254706, + "tokens_seen": 2105344000 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036826262626262624, + "loss": 1.5983, + "theoretical_loss": 3.4158079628472153, + "tokens_seen": 2105409536 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036824242424242425, + "loss": 1.37, + "theoretical_loss": 3.415799187118619, + "tokens_seen": 2105475072 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036822222222222226, + "loss": 1.2671, + "theoretical_loss": 3.415790411739658, + "tokens_seen": 2105540608 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.5804007649421692, + "objective/train/avg_token_score": 0.0030669576954096556, + "objective/train/docs_used": 1190193, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9711037874221802, + "objective/train/original_loss": 3.844700574874878, + "objective/train/theoretical_loss": 3.4157816367103067, + "objective/train/tokens_used": 464465376, + "objective/train/unlikelihood_loss": 0.07488252967596054, + "theoretical_loss": 3.4157816367103067, + "tokens_seen": 2105606144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003682020202020202, + "loss": 1.6455, + "theoretical_loss": 3.4157816367103067, + "tokens_seen": 2105606144 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036818181818181817, + "loss": 1.5489, + "theoretical_loss": 3.41577286203054, + "tokens_seen": 2105671680 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036816161616161613, + "loss": 1.4038, + "theoretical_loss": 3.4157640877003335, + "tokens_seen": 2105737216 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036814141414141414, + "loss": 1.439, + "theoretical_loss": 3.415755313719662, + "tokens_seen": 2105802752 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036812121212121215, + "loss": 1.5887, + "theoretical_loss": 3.4157465400885014, + "tokens_seen": 2105868288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681010101010101, + "loss": 1.3409, + "theoretical_loss": 3.415737766806826, + "tokens_seen": 2105933824 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036808080808080806, + "loss": 1.1507, + "theoretical_loss": 3.4157289938746125, + "tokens_seen": 2105999360 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680606060606061, + "loss": 1.3687, + "theoretical_loss": 3.415720221291834, + "tokens_seen": 2106064896 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680404040404041, + "loss": 1.5382, + "theoretical_loss": 3.4157114490584677, + "tokens_seen": 2106130432 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036802020202020204, + "loss": 1.5209, + "theoretical_loss": 3.4157026771744876, + "tokens_seen": 2106195968 + }, + { + "epoch": 0.27, + "learning_rate": 0.000368, + "loss": 1.3776, + "theoretical_loss": 3.415693905639869, + "tokens_seen": 2106261504 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036797979797979796, + "loss": 1.3829, + "theoretical_loss": 3.4156851344545878, + "tokens_seen": 2106327040 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036795959595959597, + "loss": 1.6695, + "theoretical_loss": 3.4156763636186183, + "tokens_seen": 2106392576 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367939393939394, + "loss": 1.5646, + "theoretical_loss": 3.4156675931319365, + "tokens_seen": 2106458112 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036791919191919194, + "loss": 1.5348, + "theoretical_loss": 3.4156588229945175, + "tokens_seen": 2106523648 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003678989898989899, + "loss": 1.4938, + "theoretical_loss": 3.415650053206336, + "tokens_seen": 2106589184 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036787878787878785, + "loss": 1.3349, + "theoretical_loss": 3.4156412837673678, + "tokens_seen": 2106654720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003678585858585859, + "loss": 1.4396, + "theoretical_loss": 3.415632514677588, + "tokens_seen": 2106720256 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036783838383838387, + "loss": 1.4398, + "theoretical_loss": 3.415623745936972, + "tokens_seen": 2106785792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036781818181818183, + "loss": 1.6979, + "theoretical_loss": 3.415614977545494, + "tokens_seen": 2106851328 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677979797979798, + "loss": 1.5001, + "theoretical_loss": 3.4156062095031308, + "tokens_seen": 2106916864 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036777777777777774, + "loss": 1.6035, + "theoretical_loss": 3.4155974418098562, + "tokens_seen": 2106982400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677575757575758, + "loss": 1.5589, + "theoretical_loss": 3.4155886744656465, + "tokens_seen": 2107047936 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036773737373737376, + "loss": 1.7005, + "theoretical_loss": 3.4155799074704762, + "tokens_seen": 2107113472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677171717171717, + "loss": 1.3172, + "theoretical_loss": 3.415571140824321, + "tokens_seen": 2107179008 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.4214320480823517, + "objective/train/avg_token_score": 0.011986571364104748, + "objective/train/docs_used": 1191719, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.412983775138855, + "objective/train/original_loss": 4.426974296569824, + "objective/train/theoretical_loss": 3.4155623745271564, + "objective/train/tokens_used": 466103776, + "objective/train/unlikelihood_loss": 0.0641813576221466, + "theoretical_loss": 3.4155623745271564, + "tokens_seen": 2107244544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676969696969697, + "loss": 1.4396, + "theoretical_loss": 3.4155623745271564, + "tokens_seen": 2107244544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676767676767677, + "loss": 1.4669, + "theoretical_loss": 3.415553608578957, + "tokens_seen": 2107310080 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676565656565657, + "loss": 1.3826, + "theoretical_loss": 3.4155448429796986, + "tokens_seen": 2107375616 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036763636363636366, + "loss": 1.3475, + "theoretical_loss": 3.415536077729356, + "tokens_seen": 2107441152 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676161616161616, + "loss": 1.572, + "theoretical_loss": 3.415527312827905, + "tokens_seen": 2107506688 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036759595959595957, + "loss": 1.3952, + "theoretical_loss": 3.4155185482753203, + "tokens_seen": 2107572224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675757575757576, + "loss": 1.3415, + "theoretical_loss": 3.4155097840715776, + "tokens_seen": 2107637760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675555555555556, + "loss": 1.5249, + "theoretical_loss": 3.415501020216652, + "tokens_seen": 2107703296 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036753535353535355, + "loss": 1.6025, + "theoretical_loss": 3.415492256710518, + "tokens_seen": 2107768832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675151515151515, + "loss": 1.657, + "theoretical_loss": 3.4154834935531526, + "tokens_seen": 2107834368 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674949494949495, + "loss": 1.297, + "theoretical_loss": 3.41547473074453, + "tokens_seen": 2107899904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674747474747475, + "loss": 1.1955, + "theoretical_loss": 3.415465968284625, + "tokens_seen": 2107965440 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674545454545455, + "loss": 1.6354, + "theoretical_loss": 3.4154572061734143, + "tokens_seen": 2108030976 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036743434343434344, + "loss": 1.145, + "theoretical_loss": 3.4154484444108717, + "tokens_seen": 2108096512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674141414141414, + "loss": 1.2601, + "theoretical_loss": 3.4154396829969733, + "tokens_seen": 2108162048 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673939393939394, + "loss": 1.305, + "theoretical_loss": 3.4154309219316943, + "tokens_seen": 2108227584 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036737373737373737, + "loss": 1.4799, + "theoretical_loss": 3.4154221612150097, + "tokens_seen": 2108293120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673535353535354, + "loss": 1.1589, + "theoretical_loss": 3.4154134008468953, + "tokens_seen": 2108358656 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036733333333333334, + "loss": 1.3173, + "theoretical_loss": 3.415404640827326, + "tokens_seen": 2108424192 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036731313131313135, + "loss": 1.5452, + "theoretical_loss": 3.4153958811562775, + "tokens_seen": 2108489728 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672929292929293, + "loss": 1.4035, + "theoretical_loss": 3.4153871218337244, + "tokens_seen": 2108555264 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036727272727272726, + "loss": 1.4753, + "theoretical_loss": 3.4153783628596424, + "tokens_seen": 2108620800 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036725252525252527, + "loss": 1.4524, + "theoretical_loss": 3.415369604234007, + "tokens_seen": 2108686336 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036723232323232323, + "loss": 1.425, + "theoretical_loss": 3.4153608459567932, + "tokens_seen": 2108751872 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036721212121212124, + "loss": 1.5262, + "theoretical_loss": 3.415352088027977, + "tokens_seen": 2108817408 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.40145406126976013, + "objective/train/avg_token_score": 0.04017619788646698, + "objective/train/docs_used": 1192553, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3082767724990845, + "objective/train/original_loss": 4.763609409332275, + "objective/train/theoretical_loss": 3.4153433304475325, + "objective/train/tokens_used": 467742176, + "objective/train/unlikelihood_loss": 0.044517241418361664, + "theoretical_loss": 3.4153433304475325, + "tokens_seen": 2108882944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671919191919192, + "loss": 1.3696, + "theoretical_loss": 3.4153433304475325, + "tokens_seen": 2108882944 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036717171717171715, + "loss": 1.2993, + "theoretical_loss": 3.415334573215436, + "tokens_seen": 2108948480 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036715151515151516, + "loss": 1.4364, + "theoretical_loss": 3.4153258163316624, + "tokens_seen": 2109014016 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671313131313132, + "loss": 1.5369, + "theoretical_loss": 3.4153170597961875, + "tokens_seen": 2109079552 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036711111111111113, + "loss": 1.3395, + "theoretical_loss": 3.4153083036089855, + "tokens_seen": 2109145088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670909090909091, + "loss": 1.2902, + "theoretical_loss": 3.415299547770033, + "tokens_seen": 2109210624 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036707070707070705, + "loss": 1.3252, + "theoretical_loss": 3.415290792279305, + "tokens_seen": 2109276160 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036705050505050506, + "loss": 1.2963, + "theoretical_loss": 3.415282037136776, + "tokens_seen": 2109341696 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036703030303030307, + "loss": 1.3236, + "theoretical_loss": 3.4152732823424223, + "tokens_seen": 2109407232 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367010101010101, + "loss": 1.5717, + "theoretical_loss": 3.415264527896219, + "tokens_seen": 2109472768 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366989898989899, + "loss": 1.4454, + "theoretical_loss": 3.4152557737981413, + "tokens_seen": 2109538304 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036696969696969694, + "loss": 1.4877, + "theoretical_loss": 3.4152470200481644, + "tokens_seen": 2109603840 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366949494949495, + "loss": 1.5366, + "theoretical_loss": 3.4152382666462637, + "tokens_seen": 2109669376 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036692929292929296, + "loss": 1.2176, + "theoretical_loss": 3.4152295135924153, + "tokens_seen": 2109734912 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003669090909090909, + "loss": 1.1481, + "theoretical_loss": 3.4152207608865934, + "tokens_seen": 2109800448 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668888888888889, + "loss": 1.5397, + "theoretical_loss": 3.4152120085287745, + "tokens_seen": 2109865984 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036686868686868683, + "loss": 1.5205, + "theoretical_loss": 3.4152032565189323, + "tokens_seen": 2109931520 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668484848484849, + "loss": 1.7132, + "theoretical_loss": 3.415194504857044, + "tokens_seen": 2109997056 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036682828282828285, + "loss": 1.2513, + "theoretical_loss": 3.415185753543084, + "tokens_seen": 2110062592 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668080808080808, + "loss": 1.2136, + "theoretical_loss": 3.415177002577028, + "tokens_seen": 2110128128 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036678787878787877, + "loss": 1.4298, + "theoretical_loss": 3.4151682519588507, + "tokens_seen": 2110193664 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667676767676767, + "loss": 1.468, + "theoretical_loss": 3.415159501688528, + "tokens_seen": 2110259200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667474747474748, + "loss": 1.5529, + "theoretical_loss": 3.4151507517660358, + "tokens_seen": 2110324736 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036672727272727275, + "loss": 1.1773, + "theoretical_loss": 3.4151420021913483, + "tokens_seen": 2110390272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667070707070707, + "loss": 1.4371, + "theoretical_loss": 3.415133252964442, + "tokens_seen": 2110455808 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.3256353735923767, + "objective/train/avg_token_score": 0.0284919124096632, + "objective/train/docs_used": 1194000, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0299172401428223, + "objective/train/original_loss": 4.219465255737305, + "objective/train/theoretical_loss": 3.4151245040852913, + "objective/train/tokens_used": 469380576, + "objective/train/unlikelihood_loss": 0.08671543002128601, + "theoretical_loss": 3.4151245040852913, + "tokens_seen": 2110521344 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036668686868686866, + "loss": 1.2984, + "theoretical_loss": 3.4151245040852913, + "tokens_seen": 2110521344 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036666666666666667, + "loss": 1.3646, + "theoretical_loss": 3.415115755553872, + "tokens_seen": 2110586880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666464646464647, + "loss": 1.3163, + "theoretical_loss": 3.4151070073701595, + "tokens_seen": 2110652416 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036662626262626264, + "loss": 1.4469, + "theoretical_loss": 3.4150982595341297, + "tokens_seen": 2110717952 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666060606060606, + "loss": 1.5699, + "theoretical_loss": 3.415089512045757, + "tokens_seen": 2110783488 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036658585858585855, + "loss": 1.5055, + "theoretical_loss": 3.415080764905017, + "tokens_seen": 2110849024 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036656565656565656, + "loss": 1.4742, + "theoretical_loss": 3.415072018111886, + "tokens_seen": 2110914560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665454545454546, + "loss": 1.5452, + "theoretical_loss": 3.4150632716663387, + "tokens_seen": 2110980096 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036652525252525253, + "loss": 1.668, + "theoretical_loss": 3.41505452556835, + "tokens_seen": 2111045632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665050505050505, + "loss": 1.3006, + "theoretical_loss": 3.4150457798178966, + "tokens_seen": 2111111168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664848484848485, + "loss": 1.4793, + "theoretical_loss": 3.4150370344149525, + "tokens_seen": 2111176704 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036646464646464646, + "loss": 1.1601, + "theoretical_loss": 3.415028289359494, + "tokens_seen": 2111242240 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036644444444444447, + "loss": 1.4038, + "theoretical_loss": 3.4150195446514964, + "tokens_seen": 2111307776 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664242424242424, + "loss": 1.2005, + "theoretical_loss": 3.415010800290935, + "tokens_seen": 2111373312 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664040404040404, + "loss": 1.4745, + "theoretical_loss": 3.4150020562777854, + "tokens_seen": 2111438848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663838383838384, + "loss": 1.3027, + "theoretical_loss": 3.4149933126120224, + "tokens_seen": 2111504384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663636363636364, + "loss": 1.3869, + "theoretical_loss": 3.4149845692936216, + "tokens_seen": 2111569920 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036634343434343436, + "loss": 1.3708, + "theoretical_loss": 3.414975826322559, + "tokens_seen": 2111635456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663232323232323, + "loss": 1.4073, + "theoretical_loss": 3.4149670836988095, + "tokens_seen": 2111700992 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036630303030303033, + "loss": 1.3876, + "theoretical_loss": 3.4149583414223486, + "tokens_seen": 2111766528 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662828282828283, + "loss": 1.3682, + "theoretical_loss": 3.414949599493152, + "tokens_seen": 2111832064 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662626262626263, + "loss": 1.1863, + "theoretical_loss": 3.414940857911195, + "tokens_seen": 2111897600 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036624242424242425, + "loss": 1.6353, + "theoretical_loss": 3.4149321166764532, + "tokens_seen": 2111963136 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662222222222222, + "loss": 1.2615, + "theoretical_loss": 3.4149233757889013, + "tokens_seen": 2112028672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662020202020202, + "loss": 1.4495, + "theoretical_loss": 3.4149146352485156, + "tokens_seen": 2112094208 + }, + { + "epoch": 0.27, + "objective/train/aligned_token_count": 0.43310117721557617, + "objective/train/avg_token_score": 0.03492316976189613, + "objective/train/docs_used": 1194755, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5198599100112915, + "objective/train/original_loss": 4.662668704986572, + "objective/train/theoretical_loss": 3.414905895055271, + "objective/train/tokens_used": 471018976, + "objective/train/unlikelihood_loss": 0.052197135984897614, + "theoretical_loss": 3.414905895055271, + "tokens_seen": 2112159744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661818181818182, + "loss": 1.4197, + "theoretical_loss": 3.414905895055271, + "tokens_seen": 2112159744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661616161616162, + "loss": 1.3304, + "theoretical_loss": 3.414897155209143, + "tokens_seen": 2112225280 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036614141414141415, + "loss": 1.4019, + "theoretical_loss": 3.4148884157101076, + "tokens_seen": 2112290816 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036612121212121216, + "loss": 1.6798, + "theoretical_loss": 3.4148796765581397, + "tokens_seen": 2112356352 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661010101010101, + "loss": 1.1708, + "theoretical_loss": 3.414870937753215, + "tokens_seen": 2112421888 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036608080808080807, + "loss": 1.1303, + "theoretical_loss": 3.4148621992953085, + "tokens_seen": 2112487424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660606060606061, + "loss": 1.4747, + "theoretical_loss": 3.4148534611843964, + "tokens_seen": 2112552960 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036604040404040404, + "loss": 1.4416, + "theoretical_loss": 3.414844723420453, + "tokens_seen": 2112618496 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036602020202020205, + "loss": 1.5186, + "theoretical_loss": 3.414835986003455, + "tokens_seen": 2112684032 + }, + { + "epoch": 0.28, + "learning_rate": 0.000366, + "loss": 1.3175, + "theoretical_loss": 3.4148272489333773, + "tokens_seen": 2112749568 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036597979797979796, + "loss": 1.388, + "theoretical_loss": 3.4148185122101955, + "tokens_seen": 2112815104 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365959595959596, + "loss": 1.5015, + "theoretical_loss": 3.414809775833885, + "tokens_seen": 2112880640 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365939393939394, + "loss": 1.3173, + "theoretical_loss": 3.414801039804421, + "tokens_seen": 2112946176 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036591919191919194, + "loss": 1.4154, + "theoretical_loss": 3.4147923041217796, + "tokens_seen": 2113011712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658989898989899, + "loss": 1.5216, + "theoretical_loss": 3.4147835687859356, + "tokens_seen": 2113077248 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036587878787878786, + "loss": 1.3413, + "theoretical_loss": 3.414774833796865, + "tokens_seen": 2113142784 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036585858585858587, + "loss": 1.3983, + "theoretical_loss": 3.414766099154543, + "tokens_seen": 2113208320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658383838383839, + "loss": 1.3596, + "theoretical_loss": 3.414757364858945, + "tokens_seen": 2113273856 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036581818181818183, + "loss": 1.4247, + "theoretical_loss": 3.414748630910047, + "tokens_seen": 2113339392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657979797979798, + "loss": 1.4082, + "theoretical_loss": 3.414739897307824, + "tokens_seen": 2113404928 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036577777777777775, + "loss": 1.5664, + "theoretical_loss": 3.4147311640522515, + "tokens_seen": 2113470464 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657575757575758, + "loss": 1.3231, + "theoretical_loss": 3.4147224311433053, + "tokens_seen": 2113536000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036573737373737377, + "loss": 1.3211, + "theoretical_loss": 3.41471369858096, + "tokens_seen": 2113601536 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036571717171717173, + "loss": 1.4833, + "theoretical_loss": 3.4147049663651927, + "tokens_seen": 2113667072 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656969696969697, + "loss": 1.4256, + "theoretical_loss": 3.4146962344959775, + "tokens_seen": 2113732608 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.4935850501060486, + "objective/train/avg_token_score": 0.02954690158367157, + "objective/train/docs_used": 1195575, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5347603559494019, + "objective/train/original_loss": 4.009374141693115, + "objective/train/theoretical_loss": 3.4146875029732904, + "objective/train/tokens_used": 472657376, + "objective/train/unlikelihood_loss": 0.07271783798933029, + "theoretical_loss": 3.4146875029732904, + "tokens_seen": 2113798144 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036567676767676764, + "loss": 1.3484, + "theoretical_loss": 3.4146875029732904, + "tokens_seen": 2113798144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656565656565657, + "loss": 1.5824, + "theoretical_loss": 3.4146787717971074, + "tokens_seen": 2113863680 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036563636363636366, + "loss": 1.3774, + "theoretical_loss": 3.414670040967403, + "tokens_seen": 2113929216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656161616161616, + "loss": 1.5458, + "theoretical_loss": 3.4146613104841532, + "tokens_seen": 2113994752 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655959595959596, + "loss": 1.2095, + "theoretical_loss": 3.414652580347334, + "tokens_seen": 2114060288 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655757575757576, + "loss": 1.4644, + "theoretical_loss": 3.41464385055692, + "tokens_seen": 2114125824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655555555555556, + "loss": 1.4387, + "theoretical_loss": 3.4146351211128874, + "tokens_seen": 2114191360 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036553535353535356, + "loss": 1.2225, + "theoretical_loss": 3.4146263920152116, + "tokens_seen": 2114256896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655151515151515, + "loss": 1.3472, + "theoretical_loss": 3.4146176632638676, + "tokens_seen": 2114322432 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036549494949494947, + "loss": 1.7032, + "theoretical_loss": 3.4146089348588315, + "tokens_seen": 2114387968 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654747474747475, + "loss": 1.5132, + "theoretical_loss": 3.4146002068000785, + "tokens_seen": 2114453504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654545454545455, + "loss": 1.4697, + "theoretical_loss": 3.4145914790875844, + "tokens_seen": 2114519040 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036543434343434345, + "loss": 1.3938, + "theoretical_loss": 3.414582751721325, + "tokens_seen": 2114584576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654141414141414, + "loss": 1.4168, + "theoretical_loss": 3.4145740247012752, + "tokens_seen": 2114650112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653939393939394, + "loss": 1.341, + "theoretical_loss": 3.414565298027411, + "tokens_seen": 2114715648 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653737373737374, + "loss": 1.3186, + "theoretical_loss": 3.4145565716997073, + "tokens_seen": 2114781184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653535353535354, + "loss": 1.0649, + "theoretical_loss": 3.41454784571814, + "tokens_seen": 2114846720 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036533333333333334, + "loss": 1.3758, + "theoretical_loss": 3.4145391200826856, + "tokens_seen": 2114912256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653131313131313, + "loss": 1.4244, + "theoretical_loss": 3.414530394793318, + "tokens_seen": 2114977792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652929292929293, + "loss": 1.3469, + "theoretical_loss": 3.4145216698500134, + "tokens_seen": 2115043328 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036527272727272727, + "loss": 1.2952, + "theoretical_loss": 3.414512945252748, + "tokens_seen": 2115108864 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652525252525253, + "loss": 1.7349, + "theoretical_loss": 3.414504221001496, + "tokens_seen": 2115174400 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036523232323232323, + "loss": 1.4591, + "theoretical_loss": 3.414495497096235, + "tokens_seen": 2115239936 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036521212121212124, + "loss": 1.519, + "theoretical_loss": 3.4144867735369386, + "tokens_seen": 2115305472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003651919191919192, + "loss": 1.4488, + "theoretical_loss": 3.414478050323583, + "tokens_seen": 2115371008 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.386424720287323, + "objective/train/avg_token_score": 0.013556824065744877, + "objective/train/docs_used": 1196855, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2169398069381714, + "objective/train/original_loss": 4.470401763916016, + "objective/train/theoretical_loss": 3.4144693274561444, + "objective/train/tokens_used": 474295776, + "objective/train/unlikelihood_loss": 0.04746641591191292, + "theoretical_loss": 3.4144693274561444, + "tokens_seen": 2115436544 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036517171717171716, + "loss": 1.2, + "theoretical_loss": 3.4144693274561444, + "tokens_seen": 2115436544 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036515151515151517, + "loss": 1.2002, + "theoretical_loss": 3.414460604934597, + "tokens_seen": 2115502080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003651313131313131, + "loss": 1.3827, + "theoretical_loss": 3.414451882758918, + "tokens_seen": 2115567616 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036511111111111114, + "loss": 1.5522, + "theoretical_loss": 3.414443160929082, + "tokens_seen": 2115633152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650909090909091, + "loss": 1.5936, + "theoretical_loss": 3.4144344394450643, + "tokens_seen": 2115698688 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036507070707070705, + "loss": 1.4197, + "theoretical_loss": 3.414425718306841, + "tokens_seen": 2115764224 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036505050505050506, + "loss": 1.4312, + "theoretical_loss": 3.4144169975143885, + "tokens_seen": 2115829760 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365030303030303, + "loss": 1.5033, + "theoretical_loss": 3.4144082770676807, + "tokens_seen": 2115895296 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036501010101010103, + "loss": 1.4611, + "theoretical_loss": 3.4143995569666936, + "tokens_seen": 2115960832 + }, + { + "epoch": 0.28, + "learning_rate": 0.000364989898989899, + "loss": 1.4207, + "theoretical_loss": 3.4143908372114042, + "tokens_seen": 2116026368 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036496969696969694, + "loss": 1.4198, + "theoretical_loss": 3.4143821178017864, + "tokens_seen": 2116091904 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036494949494949496, + "loss": 1.0965, + "theoretical_loss": 3.4143733987378164, + "tokens_seen": 2116157440 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036492929292929297, + "loss": 1.189, + "theoretical_loss": 3.41436468001947, + "tokens_seen": 2116222976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649090909090909, + "loss": 1.2537, + "theoretical_loss": 3.4143559616467227, + "tokens_seen": 2116288512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648888888888889, + "loss": 1.4288, + "theoretical_loss": 3.41434724361955, + "tokens_seen": 2116354048 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036486868686868684, + "loss": 1.2104, + "theoretical_loss": 3.4143385259379277, + "tokens_seen": 2116419584 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036484848484848485, + "loss": 1.5573, + "theoretical_loss": 3.4143298086018308, + "tokens_seen": 2116485120 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036482828282828286, + "loss": 1.2042, + "theoretical_loss": 3.4143210916112356, + "tokens_seen": 2116550656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648080808080808, + "loss": 1.3208, + "theoretical_loss": 3.4143123749661175, + "tokens_seen": 2116616192 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036478787878787877, + "loss": 1.3485, + "theoretical_loss": 3.414303658666452, + "tokens_seen": 2116681728 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036476767676767673, + "loss": 1.2952, + "theoretical_loss": 3.414294942712215, + "tokens_seen": 2116747264 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647474747474748, + "loss": 1.4681, + "theoretical_loss": 3.414286227103381, + "tokens_seen": 2116812800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036472727272727275, + "loss": 1.3255, + "theoretical_loss": 3.4142775118399276, + "tokens_seen": 2116878336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647070707070707, + "loss": 1.228, + "theoretical_loss": 3.414268796921829, + "tokens_seen": 2116943872 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036468686868686867, + "loss": 1.2621, + "theoretical_loss": 3.414260082349061, + "tokens_seen": 2117009408 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.4744623601436615, + "objective/train/avg_token_score": 0.007280266378074884, + "objective/train/docs_used": 1197590, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5125036239624023, + "objective/train/original_loss": 3.8015384674072266, + "objective/train/theoretical_loss": 3.414251368121599, + "objective/train/tokens_used": 475934176, + "objective/train/unlikelihood_loss": 0.0856105238199234, + "theoretical_loss": 3.414251368121599, + "tokens_seen": 2117074944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646666666666666, + "loss": 1.3927, + "theoretical_loss": 3.414251368121599, + "tokens_seen": 2117074944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646464646464647, + "loss": 1.2886, + "theoretical_loss": 3.4142426542394197, + "tokens_seen": 2117140480 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036462626262626264, + "loss": 1.3716, + "theoretical_loss": 3.414233940702498, + "tokens_seen": 2117206016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646060606060606, + "loss": 1.3096, + "theoretical_loss": 3.4142252275108094, + "tokens_seen": 2117271552 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036458585858585856, + "loss": 1.4137, + "theoretical_loss": 3.4142165146643295, + "tokens_seen": 2117337088 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645656565656566, + "loss": 1.5802, + "theoretical_loss": 3.4142078021630344, + "tokens_seen": 2117402624 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645454545454546, + "loss": 1.5419, + "theoretical_loss": 3.4141990900068997, + "tokens_seen": 2117468160 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036452525252525254, + "loss": 1.4169, + "theoretical_loss": 3.4141903781959004, + "tokens_seen": 2117533696 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645050505050505, + "loss": 1.5674, + "theoretical_loss": 3.4141816667300127, + "tokens_seen": 2117599232 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036448484848484845, + "loss": 1.2841, + "theoretical_loss": 3.414172955609212, + "tokens_seen": 2117664768 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644646464646465, + "loss": 1.3084, + "theoretical_loss": 3.414164244833475, + "tokens_seen": 2117730304 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036444444444444447, + "loss": 1.271, + "theoretical_loss": 3.4141555344027754, + "tokens_seen": 2117795840 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036442424242424243, + "loss": 1.4627, + "theoretical_loss": 3.41414682431709, + "tokens_seen": 2117861376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644040404040404, + "loss": 1.4575, + "theoretical_loss": 3.414138114576395, + "tokens_seen": 2117926912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643838383838384, + "loss": 1.5042, + "theoretical_loss": 3.414129405180665, + "tokens_seen": 2117992448 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643636363636364, + "loss": 1.5975, + "theoretical_loss": 3.414120696129876, + "tokens_seen": 2118057984 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036434343434343437, + "loss": 1.3464, + "theoretical_loss": 3.4141119874240036, + "tokens_seen": 2118123520 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643232323232323, + "loss": 1.4756, + "theoretical_loss": 3.414103279063024, + "tokens_seen": 2118189056 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643030303030303, + "loss": 1.6128, + "theoretical_loss": 3.4140945710469124, + "tokens_seen": 2118254592 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642828282828283, + "loss": 1.454, + "theoretical_loss": 3.414085863375644, + "tokens_seen": 2118320128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642626262626263, + "loss": 1.4898, + "theoretical_loss": 3.4140771560491956, + "tokens_seen": 2118385664 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036424242424242426, + "loss": 1.3478, + "theoretical_loss": 3.4140684490675426, + "tokens_seen": 2118451200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642222222222222, + "loss": 1.5727, + "theoretical_loss": 3.41405974243066, + "tokens_seen": 2118516736 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642020202020202, + "loss": 1.5289, + "theoretical_loss": 3.414051036138524, + "tokens_seen": 2118582272 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641818181818182, + "loss": 1.5049, + "theoretical_loss": 3.41404233019111, + "tokens_seen": 2118647808 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.4476417303085327, + "objective/train/avg_token_score": 0.011445474810898304, + "objective/train/docs_used": 1199039, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3598164319992065, + "objective/train/original_loss": 3.9978601932525635, + "objective/train/theoretical_loss": 3.4140336245883933, + "objective/train/tokens_used": 477572576, + "objective/train/unlikelihood_loss": 0.08685135096311569, + "theoretical_loss": 3.4140336245883933, + "tokens_seen": 2118713344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641616161616162, + "loss": 1.2644, + "theoretical_loss": 3.4140336245883933, + "tokens_seen": 2118713344 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036414141414141415, + "loss": 1.7291, + "theoretical_loss": 3.414024919330351, + "tokens_seen": 2118778880 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641212121212121, + "loss": 1.3314, + "theoretical_loss": 3.4140162144169577, + "tokens_seen": 2118844416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641010101010101, + "loss": 1.3058, + "theoretical_loss": 3.4140075098481892, + "tokens_seen": 2118909952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640808080808081, + "loss": 1.3949, + "theoretical_loss": 3.4139988056240216, + "tokens_seen": 2118975488 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640606060606061, + "loss": 1.2031, + "theoretical_loss": 3.41399010174443, + "tokens_seen": 2119041024 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036404040404040404, + "loss": 1.3957, + "theoretical_loss": 3.4139813982093905, + "tokens_seen": 2119106560 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036402020202020205, + "loss": 1.51, + "theoretical_loss": 3.413972695018879, + "tokens_seen": 2119172096 + }, + { + "epoch": 0.28, + "learning_rate": 0.000364, + "loss": 1.5684, + "theoretical_loss": 3.413963992172871, + "tokens_seen": 2119237632 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036397979797979797, + "loss": 1.2647, + "theoretical_loss": 3.4139552896713417, + "tokens_seen": 2119303168 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363959595959596, + "loss": 1.4358, + "theoretical_loss": 3.4139465875142676, + "tokens_seen": 2119368704 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036393939393939394, + "loss": 1.6356, + "theoretical_loss": 3.413937885701624, + "tokens_seen": 2119434240 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036391919191919195, + "loss": 1.4228, + "theoretical_loss": 3.4139291842333863, + "tokens_seen": 2119499776 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638989898989899, + "loss": 1.3778, + "theoretical_loss": 3.413920483109531, + "tokens_seen": 2119565312 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036387878787878786, + "loss": 1.4368, + "theoretical_loss": 3.413911782330034, + "tokens_seen": 2119630848 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036385858585858587, + "loss": 1.4418, + "theoretical_loss": 3.41390308189487, + "tokens_seen": 2119696384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638383838383839, + "loss": 1.3635, + "theoretical_loss": 3.413894381804015, + "tokens_seen": 2119761920 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036381818181818184, + "loss": 1.4966, + "theoretical_loss": 3.413885682057445, + "tokens_seen": 2119827456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637979797979798, + "loss": 1.5134, + "theoretical_loss": 3.413876982655135, + "tokens_seen": 2119892992 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036377777777777775, + "loss": 1.1802, + "theoretical_loss": 3.4138682835970626, + "tokens_seen": 2119958528 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036375757575757576, + "loss": 1.3945, + "theoretical_loss": 3.413859584883202, + "tokens_seen": 2120024064 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637373737373738, + "loss": 1.3595, + "theoretical_loss": 3.413850886513529, + "tokens_seen": 2120089600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036371717171717173, + "loss": 1.4383, + "theoretical_loss": 3.4138421884880197, + "tokens_seen": 2120155136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636969696969697, + "loss": 1.3119, + "theoretical_loss": 3.41383349080665, + "tokens_seen": 2120220672 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036367676767676765, + "loss": 1.356, + "theoretical_loss": 3.413824793469395, + "tokens_seen": 2120286208 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.3866690993309021, + "objective/train/avg_token_score": 0.043729815632104874, + "objective/train/docs_used": 1199714, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2710665464401245, + "objective/train/original_loss": 4.90679931640625, + "objective/train/theoretical_loss": 3.4138160964762307, + "objective/train/tokens_used": 479210976, + "objective/train/unlikelihood_loss": 0.06792816519737244, + "theoretical_loss": 3.4138160964762307, + "tokens_seen": 2120351744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636565656565657, + "loss": 1.3698, + "theoretical_loss": 3.4138160964762307, + "tokens_seen": 2120351744 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036363636363636367, + "loss": 1.9067, + "theoretical_loss": 3.4138073998271334, + "tokens_seen": 2120417280 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636161616161616, + "loss": 1.2395, + "theoretical_loss": 3.4137987035220787, + "tokens_seen": 2120482816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003635959595959596, + "loss": 1.3265, + "theoretical_loss": 3.4137900075610412, + "tokens_seen": 2120548352 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036357575757575754, + "loss": 1.2914, + "theoretical_loss": 3.413781311943999, + "tokens_seen": 2120613888 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003635555555555556, + "loss": 1.4336, + "theoretical_loss": 3.413772616670925, + "tokens_seen": 2120679424 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036353535353535356, + "loss": 1.5028, + "theoretical_loss": 3.413763921741797, + "tokens_seen": 2120744960 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003635151515151515, + "loss": 1.5701, + "theoretical_loss": 3.4137552271565905, + "tokens_seen": 2120810496 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634949494949495, + "loss": 1.4403, + "theoretical_loss": 3.4137465329152805, + "tokens_seen": 2120876032 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036347474747474743, + "loss": 1.3683, + "theoretical_loss": 3.4137378390178434, + "tokens_seen": 2120941568 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634545454545455, + "loss": 1.4095, + "theoretical_loss": 3.413729145464255, + "tokens_seen": 2121007104 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036343434343434345, + "loss": 1.5156, + "theoretical_loss": 3.4137204522544904, + "tokens_seen": 2121072640 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634141414141414, + "loss": 1.4724, + "theoretical_loss": 3.413711759388526, + "tokens_seen": 2121138176 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036339393939393937, + "loss": 1.2512, + "theoretical_loss": 3.4137030668663373, + "tokens_seen": 2121203712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633737373737374, + "loss": 1.4421, + "theoretical_loss": 3.4136943746879007, + "tokens_seen": 2121269248 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633535353535354, + "loss": 1.3767, + "theoretical_loss": 3.413685682853191, + "tokens_seen": 2121334784 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036333333333333335, + "loss": 1.1784, + "theoretical_loss": 3.413676991362185, + "tokens_seen": 2121400320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633131313131313, + "loss": 1.2975, + "theoretical_loss": 3.413668300214858, + "tokens_seen": 2121465856 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036329292929292926, + "loss": 1.3971, + "theoretical_loss": 3.413659609411185, + "tokens_seen": 2121531392 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036327272727272727, + "loss": 1.5969, + "theoretical_loss": 3.4136509189511433, + "tokens_seen": 2121596928 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632525252525253, + "loss": 1.4649, + "theoretical_loss": 3.4136422288347075, + "tokens_seen": 2121662464 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036323232323232324, + "loss": 1.3678, + "theoretical_loss": 3.4136335390618537, + "tokens_seen": 2121728000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632121212121212, + "loss": 1.1506, + "theoretical_loss": 3.4136248496325585, + "tokens_seen": 2121793536 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631919191919192, + "loss": 1.4502, + "theoretical_loss": 3.413616160546797, + "tokens_seen": 2121859072 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036317171717171716, + "loss": 1.3311, + "theoretical_loss": 3.4136074718045446, + "tokens_seen": 2121924608 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.31140029430389404, + "objective/train/avg_token_score": 0.013784058392047882, + "objective/train/docs_used": 1200493, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9349797964096069, + "objective/train/original_loss": 4.567559242248535, + "objective/train/theoretical_loss": 3.413598783405778, + "objective/train/tokens_used": 480849376, + "objective/train/unlikelihood_loss": 0.0624493733048439, + "theoretical_loss": 3.413598783405778, + "tokens_seen": 2121990144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631515151515152, + "loss": 1.4248, + "theoretical_loss": 3.413598783405778, + "tokens_seen": 2121990144 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036313131313131313, + "loss": 1.2625, + "theoretical_loss": 3.4135900953504725, + "tokens_seen": 2122055680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631111111111111, + "loss": 1.2486, + "theoretical_loss": 3.4135814076386044, + "tokens_seen": 2122121216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003630909090909091, + "loss": 1.3548, + "theoretical_loss": 3.4135727202701487, + "tokens_seen": 2122186752 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036307070707070706, + "loss": 1.6199, + "theoretical_loss": 3.4135640332450814, + "tokens_seen": 2122252288 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036305050505050507, + "loss": 1.5875, + "theoretical_loss": 3.4135553465633794, + "tokens_seen": 2122317824 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363030303030303, + "loss": 1.5659, + "theoretical_loss": 3.413546660225017, + "tokens_seen": 2122383360 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036301010101010104, + "loss": 1.4644, + "theoretical_loss": 3.413537974229971, + "tokens_seen": 2122448896 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362989898989899, + "loss": 1.3691, + "theoretical_loss": 3.413529288578217, + "tokens_seen": 2122514432 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036296969696969695, + "loss": 1.4405, + "theoretical_loss": 3.4135206032697307, + "tokens_seen": 2122579968 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036294949494949496, + "loss": 1.4858, + "theoretical_loss": 3.413511918304488, + "tokens_seen": 2122645504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629292929292929, + "loss": 1.4858, + "theoretical_loss": 3.413503233682465, + "tokens_seen": 2122711040 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036290909090909093, + "loss": 1.364, + "theoretical_loss": 3.4134945494036373, + "tokens_seen": 2122776576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628888888888889, + "loss": 1.4413, + "theoretical_loss": 3.4134858654679805, + "tokens_seen": 2122842112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628686868686869, + "loss": 1.4418, + "theoretical_loss": 3.4134771818754714, + "tokens_seen": 2122907648 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036284848484848485, + "loss": 1.3524, + "theoretical_loss": 3.4134684986260844, + "tokens_seen": 2122973184 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036282828282828286, + "loss": 1.3216, + "theoretical_loss": 3.413459815719796, + "tokens_seen": 2123038720 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628080808080808, + "loss": 1.2786, + "theoretical_loss": 3.413451133156583, + "tokens_seen": 2123104256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627878787878788, + "loss": 1.3786, + "theoretical_loss": 3.41344245093642, + "tokens_seen": 2123169792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627676767676768, + "loss": 1.1093, + "theoretical_loss": 3.4134337690592833, + "tokens_seen": 2123235328 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036274747474747475, + "loss": 1.4715, + "theoretical_loss": 3.413425087525149, + "tokens_seen": 2123300864 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036272727272727276, + "loss": 1.5278, + "theoretical_loss": 3.413416406333992, + "tokens_seen": 2123366400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627070707070707, + "loss": 1.3447, + "theoretical_loss": 3.413407725485789, + "tokens_seen": 2123431936 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036268686868686867, + "loss": 1.3902, + "theoretical_loss": 3.4133990449805163, + "tokens_seen": 2123497472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626666666666667, + "loss": 1.4581, + "theoretical_loss": 3.4133903648181487, + "tokens_seen": 2123563008 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.46126589179039, + "objective/train/avg_token_score": 0.017873598262667656, + "objective/train/docs_used": 1201963, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.436392903327942, + "objective/train/original_loss": 4.299962043762207, + "objective/train/theoretical_loss": 3.413381684998663, + "objective/train/tokens_used": 482487776, + "objective/train/unlikelihood_loss": 0.07796698063611984, + "theoretical_loss": 3.413381684998663, + "tokens_seen": 2123628544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626464646464647, + "loss": 1.3903, + "theoretical_loss": 3.413381684998663, + "tokens_seen": 2123628544 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036262626262626265, + "loss": 1.5286, + "theoretical_loss": 3.4133730055220344, + "tokens_seen": 2123694080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626060606060606, + "loss": 1.5028, + "theoretical_loss": 3.4133643263882387, + "tokens_seen": 2123759616 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036258585858585856, + "loss": 1.4162, + "theoretical_loss": 3.413355647597253, + "tokens_seen": 2123825152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625656565656566, + "loss": 1.4244, + "theoretical_loss": 3.4133469691490514, + "tokens_seen": 2123890688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625454545454546, + "loss": 1.8553, + "theoretical_loss": 3.413338291043611, + "tokens_seen": 2123956224 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036252525252525254, + "loss": 1.3402, + "theoretical_loss": 3.4133296132809074, + "tokens_seen": 2124021760 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625050505050505, + "loss": 1.3649, + "theoretical_loss": 3.4133209358609164, + "tokens_seen": 2124087296 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036248484848484846, + "loss": 1.3512, + "theoretical_loss": 3.413312258783614, + "tokens_seen": 2124152832 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624646464646465, + "loss": 1.2718, + "theoretical_loss": 3.413303582048976, + "tokens_seen": 2124218368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624444444444445, + "loss": 1.498, + "theoretical_loss": 3.4132949056569784, + "tokens_seen": 2124283904 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036242424242424244, + "loss": 1.4112, + "theoretical_loss": 3.4132862296075968, + "tokens_seen": 2124349440 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624040404040404, + "loss": 1.38, + "theoretical_loss": 3.413277553900808, + "tokens_seen": 2124414976 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036238383838383835, + "loss": 1.3447, + "theoretical_loss": 3.413268878536586, + "tokens_seen": 2124480512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623636363636364, + "loss": 1.5402, + "theoretical_loss": 3.413260203514909, + "tokens_seen": 2124546048 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036234343434343437, + "loss": 1.4928, + "theoretical_loss": 3.413251528835751, + "tokens_seen": 2124611584 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036232323232323233, + "loss": 1.3285, + "theoretical_loss": 3.4132428544990896, + "tokens_seen": 2124677120 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623030303030303, + "loss": 1.2969, + "theoretical_loss": 3.4132341805048996, + "tokens_seen": 2124742656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622828282828283, + "loss": 1.5159, + "theoretical_loss": 3.4132255068531574, + "tokens_seen": 2124808192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622626262626263, + "loss": 1.5201, + "theoretical_loss": 3.413216833543838, + "tokens_seen": 2124873728 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036224242424242426, + "loss": 1.2455, + "theoretical_loss": 3.413208160576919, + "tokens_seen": 2124939264 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622222222222222, + "loss": 1.436, + "theoretical_loss": 3.413199487952374, + "tokens_seen": 2125004800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622020202020202, + "loss": 1.4685, + "theoretical_loss": 3.4131908156701813, + "tokens_seen": 2125070336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621818181818182, + "loss": 1.4574, + "theoretical_loss": 3.4131821437303156, + "tokens_seen": 2125135872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621616161616162, + "loss": 1.4075, + "theoretical_loss": 3.4131734721327533, + "tokens_seen": 2125201408 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.5243768095970154, + "objective/train/avg_token_score": 0.016918325796723366, + "objective/train/docs_used": 1202675, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5344035625457764, + "objective/train/original_loss": 4.4746246337890625, + "objective/train/theoretical_loss": 3.41316480087747, + "objective/train/tokens_used": 484126176, + "objective/train/unlikelihood_loss": 0.03440330550074577, + "theoretical_loss": 3.41316480087747, + "tokens_seen": 2125266944 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036214141414141416, + "loss": 1.4035, + "theoretical_loss": 3.41316480087747, + "tokens_seen": 2125266944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621212121212121, + "loss": 1.4559, + "theoretical_loss": 3.4131561299644417, + "tokens_seen": 2125332480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621010101010101, + "loss": 1.4123, + "theoretical_loss": 3.413147459393644, + "tokens_seen": 2125398016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620808080808081, + "loss": 1.5865, + "theoretical_loss": 3.4131387891650533, + "tokens_seen": 2125463552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620606060606061, + "loss": 1.4184, + "theoretical_loss": 3.4131301192786454, + "tokens_seen": 2125529088 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036204040404040405, + "loss": 1.3117, + "theoretical_loss": 3.4131214497343967, + "tokens_seen": 2125594624 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362020202020202, + "loss": 1.1812, + "theoretical_loss": 3.4131127805322823, + "tokens_seen": 2125660160 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362, + "loss": 1.3233, + "theoretical_loss": 3.4131041116722787, + "tokens_seen": 2125725696 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361979797979798, + "loss": 1.5804, + "theoretical_loss": 3.413095443154362, + "tokens_seen": 2125791232 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361959595959596, + "loss": 1.3498, + "theoretical_loss": 3.4130867749785074, + "tokens_seen": 2125856768 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036193939393939394, + "loss": 1.4536, + "theoretical_loss": 3.4130781071446914, + "tokens_seen": 2125922304 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003619191919191919, + "loss": 1.5518, + "theoretical_loss": 3.4130694396528902, + "tokens_seen": 2125987840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618989898989899, + "loss": 1.494, + "theoretical_loss": 3.413060772503079, + "tokens_seen": 2126053376 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036187878787878787, + "loss": 1.244, + "theoretical_loss": 3.4130521056952348, + "tokens_seen": 2126118912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618585858585859, + "loss": 1.5502, + "theoretical_loss": 3.413043439229333, + "tokens_seen": 2126184448 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036183838383838383, + "loss": 1.2781, + "theoretical_loss": 3.413034773105349, + "tokens_seen": 2126249984 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036181818181818185, + "loss": 1.5193, + "theoretical_loss": 3.41302610732326, + "tokens_seen": 2126315520 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003617979797979798, + "loss": 1.2846, + "theoretical_loss": 3.4130174418830403, + "tokens_seen": 2126381056 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036177777777777776, + "loss": 1.5768, + "theoretical_loss": 3.4130087767846677, + "tokens_seen": 2126446592 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036175757575757577, + "loss": 1.714, + "theoretical_loss": 3.4130001120281173, + "tokens_seen": 2126512128 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036173737373737373, + "loss": 1.4666, + "theoretical_loss": 3.412991447613365, + "tokens_seen": 2126577664 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036171717171717174, + "loss": 1.5119, + "theoretical_loss": 3.412982783540387, + "tokens_seen": 2126643200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003616969696969697, + "loss": 1.3709, + "theoretical_loss": 3.412974119809159, + "tokens_seen": 2126708736 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036167676767676765, + "loss": 1.6, + "theoretical_loss": 3.4129654564196574, + "tokens_seen": 2126774272 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036165656565656566, + "loss": 1.327, + "theoretical_loss": 3.4129567933718583, + "tokens_seen": 2126839808 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.4833821952342987, + "objective/train/avg_token_score": 0.004767133388668299, + "objective/train/docs_used": 1203810, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5523133277893066, + "objective/train/original_loss": 3.932647705078125, + "objective/train/theoretical_loss": 3.412948130665737, + "objective/train/tokens_used": 485764576, + "objective/train/unlikelihood_loss": 0.051562901586294174, + "theoretical_loss": 3.412948130665737, + "tokens_seen": 2126905344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003616363636363637, + "loss": 1.5143, + "theoretical_loss": 3.412948130665737, + "tokens_seen": 2126905344 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036161616161616163, + "loss": 1.3193, + "theoretical_loss": 3.4129394683012704, + "tokens_seen": 2126970880 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615959595959596, + "loss": 1.4429, + "theoretical_loss": 3.4129308062784336, + "tokens_seen": 2127036416 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036157575757575754, + "loss": 1.263, + "theoretical_loss": 3.4129221445972027, + "tokens_seen": 2127101952 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036155555555555556, + "loss": 1.4309, + "theoretical_loss": 3.4129134832575545, + "tokens_seen": 2127167488 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036153535353535357, + "loss": 1.5379, + "theoretical_loss": 3.4129048222594642, + "tokens_seen": 2127233024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615151515151515, + "loss": 1.5309, + "theoretical_loss": 3.4128961616029083, + "tokens_seen": 2127298560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614949494949495, + "loss": 1.3326, + "theoretical_loss": 3.4128875012878623, + "tokens_seen": 2127364096 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036147474747474744, + "loss": 1.2591, + "theoretical_loss": 3.4128788413143027, + "tokens_seen": 2127429632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614545454545455, + "loss": 1.1578, + "theoretical_loss": 3.412870181682206, + "tokens_seen": 2127495168 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036143434343434346, + "loss": 1.4151, + "theoretical_loss": 3.412861522391547, + "tokens_seen": 2127560704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614141414141414, + "loss": 1.2281, + "theoretical_loss": 3.412852863442302, + "tokens_seen": 2127626240 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613939393939394, + "loss": 1.3528, + "theoretical_loss": 3.412844204834448, + "tokens_seen": 2127691776 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036137373737373733, + "loss": 1.1927, + "theoretical_loss": 3.4128355465679596, + "tokens_seen": 2127757312 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613535353535354, + "loss": 1.3806, + "theoretical_loss": 3.412826888642814, + "tokens_seen": 2127822848 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036133333333333335, + "loss": 1.3415, + "theoretical_loss": 3.4128182310589867, + "tokens_seen": 2127888384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613131313131313, + "loss": 1.6989, + "theoretical_loss": 3.4128095738164537, + "tokens_seen": 2127953920 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036129292929292927, + "loss": 1.6076, + "theoretical_loss": 3.4128009169151916, + "tokens_seen": 2128019456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612727272727273, + "loss": 1.6315, + "theoretical_loss": 3.4127922603551752, + "tokens_seen": 2128084992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612525252525253, + "loss": 1.2592, + "theoretical_loss": 3.412783604136382, + "tokens_seen": 2128150528 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036123232323232324, + "loss": 1.4463, + "theoretical_loss": 3.4127749482587872, + "tokens_seen": 2128216064 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612121212121212, + "loss": 1.1009, + "theoretical_loss": 3.412766292722367, + "tokens_seen": 2128281600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036119191919191916, + "loss": 1.3194, + "theoretical_loss": 3.412757637527098, + "tokens_seen": 2128347136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003611717171717172, + "loss": 1.4327, + "theoretical_loss": 3.412748982672955, + "tokens_seen": 2128412672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003611515151515152, + "loss": 1.4485, + "theoretical_loss": 3.412740328159915, + "tokens_seen": 2128478208 + }, + { + "epoch": 0.28, + "objective/train/aligned_token_count": 0.39565005898475647, + "objective/train/avg_token_score": 0.030088288709521294, + "objective/train/docs_used": 1204521, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.358384609222412, + "objective/train/original_loss": 4.55379056930542, + "objective/train/theoretical_loss": 3.4127316739879543, + "objective/train/tokens_used": 487402976, + "objective/train/unlikelihood_loss": 0.047692324966192245, + "theoretical_loss": 3.4127316739879543, + "tokens_seen": 2128543744 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036113131313131314, + "loss": 1.3064, + "theoretical_loss": 3.4127316739879543, + "tokens_seen": 2128543744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003611111111111111, + "loss": 1.4247, + "theoretical_loss": 3.4127230201570478, + "tokens_seen": 2128609280 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610909090909091, + "loss": 1.5585, + "theoretical_loss": 3.4127143666671724, + "tokens_seen": 2128674816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610707070707071, + "loss": 1.6389, + "theoretical_loss": 3.4127057135183043, + "tokens_seen": 2128740352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610505050505051, + "loss": 1.4532, + "theoretical_loss": 3.412697060710419, + "tokens_seen": 2128805888 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036103030303030303, + "loss": 1.6741, + "theoretical_loss": 3.4126884082434934, + "tokens_seen": 2128871424 + }, + { + "epoch": 0.29, + "learning_rate": 0.000361010101010101, + "loss": 1.4898, + "theoretical_loss": 3.412679756117502, + "tokens_seen": 2128936960 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360989898989899, + "loss": 1.4162, + "theoretical_loss": 3.4126711043324227, + "tokens_seen": 2129002496 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360969696969697, + "loss": 1.5396, + "theoretical_loss": 3.4126624528882306, + "tokens_seen": 2129068032 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036094949494949497, + "loss": 1.4113, + "theoretical_loss": 3.4126538017849017, + "tokens_seen": 2129133568 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609292929292929, + "loss": 1.4209, + "theoretical_loss": 3.4126451510224127, + "tokens_seen": 2129199104 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036090909090909093, + "loss": 1.4796, + "theoretical_loss": 3.4126365006007386, + "tokens_seen": 2129264640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608888888888889, + "loss": 1.5643, + "theoretical_loss": 3.412627850519857, + "tokens_seen": 2129330176 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608686868686869, + "loss": 1.5414, + "theoretical_loss": 3.412619200779743, + "tokens_seen": 2129395712 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036084848484848486, + "loss": 1.4031, + "theoretical_loss": 3.4126105513803724, + "tokens_seen": 2129461248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608282828282828, + "loss": 1.5691, + "theoretical_loss": 3.4126019023217222, + "tokens_seen": 2129526784 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608080808080808, + "loss": 1.3082, + "theoretical_loss": 3.4125932536037675, + "tokens_seen": 2129592320 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607878787878788, + "loss": 1.3075, + "theoretical_loss": 3.4125846052264857, + "tokens_seen": 2129657856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607676767676768, + "loss": 1.3246, + "theoretical_loss": 3.4125759571898513, + "tokens_seen": 2129723392 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036074747474747475, + "loss": 1.6082, + "theoretical_loss": 3.412567309493842, + "tokens_seen": 2129788928 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036072727272727276, + "loss": 1.364, + "theoretical_loss": 3.4125586621384327, + "tokens_seen": 2129854464 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607070707070707, + "loss": 1.4541, + "theoretical_loss": 3.4125500151236, + "tokens_seen": 2129920000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606868686868687, + "loss": 1.3068, + "theoretical_loss": 3.41254136844932, + "tokens_seen": 2129985536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606666666666667, + "loss": 1.4795, + "theoretical_loss": 3.412532722115569, + "tokens_seen": 2130051072 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036064646464646464, + "loss": 1.253, + "theoretical_loss": 3.412524076122322, + "tokens_seen": 2130116608 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.5087365508079529, + "objective/train/avg_token_score": 0.009626802057027817, + "objective/train/docs_used": 1206012, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.630181074142456, + "objective/train/original_loss": 3.9676575660705566, + "objective/train/theoretical_loss": 3.412515430469557, + "objective/train/tokens_used": 489041376, + "objective/train/unlikelihood_loss": 0.08346080034971237, + "theoretical_loss": 3.412515430469557, + "tokens_seen": 2130182144 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036062626262626266, + "loss": 1.5235, + "theoretical_loss": 3.412515430469557, + "tokens_seen": 2130182144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606060606060606, + "loss": 1.5016, + "theoretical_loss": 3.412506785157248, + "tokens_seen": 2130247680 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036058585858585857, + "loss": 1.5602, + "theoretical_loss": 3.4124981401853733, + "tokens_seen": 2130313216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605656565656566, + "loss": 1.456, + "theoretical_loss": 3.412489495553908, + "tokens_seen": 2130378752 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605454545454546, + "loss": 1.5841, + "theoretical_loss": 3.4124808512628277, + "tokens_seen": 2130444288 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036052525252525255, + "loss": 1.5069, + "theoretical_loss": 3.412472207312109, + "tokens_seen": 2130509824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605050505050505, + "loss": 1.3602, + "theoretical_loss": 3.4124635637017278, + "tokens_seen": 2130575360 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036048484848484846, + "loss": 1.5518, + "theoretical_loss": 3.4124549204316605, + "tokens_seen": 2130640896 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036046464646464647, + "loss": 1.5326, + "theoretical_loss": 3.412446277501884, + "tokens_seen": 2130706432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604444444444445, + "loss": 1.4563, + "theoretical_loss": 3.4124376349123726, + "tokens_seen": 2130771968 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036042424242424244, + "loss": 1.5847, + "theoretical_loss": 3.412428992663104, + "tokens_seen": 2130837504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604040404040404, + "loss": 1.7093, + "theoretical_loss": 3.4124203507540534, + "tokens_seen": 2130903040 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036038383838383835, + "loss": 1.6105, + "theoretical_loss": 3.4124117091851978, + "tokens_seen": 2130968576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003603636363636364, + "loss": 1.5254, + "theoretical_loss": 3.4124030679565127, + "tokens_seen": 2131034112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003603434343434344, + "loss": 1.482, + "theoretical_loss": 3.4123944270679742, + "tokens_seen": 2131099648 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036032323232323233, + "loss": 1.5934, + "theoretical_loss": 3.4123857865195593, + "tokens_seen": 2131165184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003603030303030303, + "loss": 1.3605, + "theoretical_loss": 3.4123771463112433, + "tokens_seen": 2131230720 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036028282828282825, + "loss": 1.4594, + "theoretical_loss": 3.412368506443002, + "tokens_seen": 2131296256 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602626262626263, + "loss": 1.4644, + "theoretical_loss": 3.412359866914813, + "tokens_seen": 2131361792 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036024242424242427, + "loss": 1.5003, + "theoretical_loss": 3.4123512277266514, + "tokens_seen": 2131427328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602222222222222, + "loss": 1.3692, + "theoretical_loss": 3.4123425888784933, + "tokens_seen": 2131492864 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602020202020202, + "loss": 1.3248, + "theoretical_loss": 3.4123339503703156, + "tokens_seen": 2131558400 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036018181818181814, + "loss": 1.5451, + "theoretical_loss": 3.4123253122020936, + "tokens_seen": 2131623936 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601616161616162, + "loss": 1.5646, + "theoretical_loss": 3.4123166743738045, + "tokens_seen": 2131689472 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036014141414141416, + "loss": 1.2047, + "theoretical_loss": 3.412308036885423, + "tokens_seen": 2131755008 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.5309139490127563, + "objective/train/avg_token_score": 0.013998433016240597, + "objective/train/docs_used": 1206751, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8272676467895508, + "objective/train/original_loss": 4.273675441741943, + "objective/train/theoretical_loss": 3.412299399736927, + "objective/train/tokens_used": 490679776, + "objective/train/unlikelihood_loss": 0.04324044659733772, + "theoretical_loss": 3.412299399736927, + "tokens_seen": 2131820544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601212121212121, + "loss": 1.5616, + "theoretical_loss": 3.412299399736927, + "tokens_seen": 2131820544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601010101010101, + "loss": 1.165, + "theoretical_loss": 3.4122907629282913, + "tokens_seen": 2131886080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600808080808081, + "loss": 1.4589, + "theoretical_loss": 3.412282126459493, + "tokens_seen": 2131951616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600606060606061, + "loss": 1.4226, + "theoretical_loss": 3.412273490330507, + "tokens_seen": 2132017152 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036004040404040405, + "loss": 1.5225, + "theoretical_loss": 3.412264854541311, + "tokens_seen": 2132082688 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360020202020202, + "loss": 1.4629, + "theoretical_loss": 3.412256219091881, + "tokens_seen": 2132148224 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035999999999999997, + "loss": 1.3966, + "theoretical_loss": 3.412247583982192, + "tokens_seen": 2132213760 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359979797979798, + "loss": 1.4012, + "theoretical_loss": 3.412238949212221, + "tokens_seen": 2132279296 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359959595959596, + "loss": 1.2143, + "theoretical_loss": 3.4122303147819446, + "tokens_seen": 2132344832 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035993939393939395, + "loss": 1.4501, + "theoretical_loss": 3.412221680691338, + "tokens_seen": 2132410368 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003599191919191919, + "loss": 1.4592, + "theoretical_loss": 3.412213046940378, + "tokens_seen": 2132475904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598989898989899, + "loss": 1.4762, + "theoretical_loss": 3.4122044135290412, + "tokens_seen": 2132541440 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035987878787878787, + "loss": 1.3354, + "theoretical_loss": 3.412195780457303, + "tokens_seen": 2132606976 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598585858585859, + "loss": 1.6177, + "theoretical_loss": 3.41218714772514, + "tokens_seen": 2132672512 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035983838383838384, + "loss": 1.5753, + "theoretical_loss": 3.412178515332528, + "tokens_seen": 2132738048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598181818181818, + "loss": 1.4519, + "theoretical_loss": 3.412169883279444, + "tokens_seen": 2132803584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597979797979798, + "loss": 1.4036, + "theoretical_loss": 3.412161251565864, + "tokens_seen": 2132869120 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035977777777777776, + "loss": 1.4998, + "theoretical_loss": 3.4121526201917636, + "tokens_seen": 2132934656 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597575757575758, + "loss": 1.5271, + "theoretical_loss": 3.4121439891571193, + "tokens_seen": 2133000192 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035973737373737373, + "loss": 1.4258, + "theoretical_loss": 3.4121353584619074, + "tokens_seen": 2133065728 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035971717171717174, + "loss": 1.2605, + "theoretical_loss": 3.412126728106104, + "tokens_seen": 2133131264 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596969696969697, + "loss": 1.1889, + "theoretical_loss": 3.412118098089686, + "tokens_seen": 2133196800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035967676767676766, + "loss": 1.4107, + "theoretical_loss": 3.4121094684126287, + "tokens_seen": 2133262336 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035965656565656567, + "loss": 1.5936, + "theoretical_loss": 3.412100839074909, + "tokens_seen": 2133327872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596363636363636, + "loss": 1.5137, + "theoretical_loss": 3.412092210076503, + "tokens_seen": 2133393408 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.2601417303085327, + "objective/train/avg_token_score": 0.018833760172128677, + "objective/train/docs_used": 1207876, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8731229305267334, + "objective/train/original_loss": 4.842316150665283, + "objective/train/theoretical_loss": 3.412083581417386, + "objective/train/tokens_used": 492318176, + "objective/train/unlikelihood_loss": 0.054733455181121826, + "theoretical_loss": 3.412083581417386, + "tokens_seen": 2133458944 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035961616161616164, + "loss": 1.3005, + "theoretical_loss": 3.412083581417386, + "tokens_seen": 2133458944 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595959595959596, + "loss": 1.4467, + "theoretical_loss": 3.412074953097536, + "tokens_seen": 2133524480 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035957575757575755, + "loss": 1.376, + "theoretical_loss": 3.412066325116928, + "tokens_seen": 2133590016 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035955555555555556, + "loss": 1.6853, + "theoretical_loss": 3.4120576974755386, + "tokens_seen": 2133655552 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035953535353535357, + "loss": 1.3437, + "theoretical_loss": 3.4120490701733432, + "tokens_seen": 2133721088 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035951515151515153, + "loss": 1.3724, + "theoretical_loss": 3.4120404432103193, + "tokens_seen": 2133786624 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594949494949495, + "loss": 1.5865, + "theoretical_loss": 3.4120318165864427, + "tokens_seen": 2133852160 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594747474747475, + "loss": 1.4658, + "theoretical_loss": 3.41202319030169, + "tokens_seen": 2133917696 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035945454545454545, + "loss": 1.4402, + "theoretical_loss": 3.4120145643560367, + "tokens_seen": 2133983232 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035943434343434347, + "loss": 1.2873, + "theoretical_loss": 3.4120059387494592, + "tokens_seen": 2134048768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594141414141414, + "loss": 1.3218, + "theoretical_loss": 3.411997313481934, + "tokens_seen": 2134114304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593939393939394, + "loss": 1.4666, + "theoretical_loss": 3.411988688553438, + "tokens_seen": 2134179840 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593737373737374, + "loss": 1.4111, + "theoretical_loss": 3.411980063963946, + "tokens_seen": 2134245376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593535353535354, + "loss": 1.4017, + "theoretical_loss": 3.4119714397134357, + "tokens_seen": 2134310912 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035933333333333336, + "loss": 1.3271, + "theoretical_loss": 3.4119628158018824, + "tokens_seen": 2134376448 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593131313131313, + "loss": 1.4653, + "theoretical_loss": 3.411954192229263, + "tokens_seen": 2134441984 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035929292929292927, + "loss": 1.4801, + "theoretical_loss": 3.411945568995553, + "tokens_seen": 2134507520 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592727272727273, + "loss": 1.4413, + "theoretical_loss": 3.411936946100729, + "tokens_seen": 2134573056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592525252525253, + "loss": 1.3266, + "theoretical_loss": 3.4119283235447684, + "tokens_seen": 2134638592 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035923232323232325, + "loss": 1.2132, + "theoretical_loss": 3.411919701327646, + "tokens_seen": 2134704128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592121212121212, + "loss": 1.1523, + "theoretical_loss": 3.4119110794493386, + "tokens_seen": 2134769664 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035919191919191916, + "loss": 1.4889, + "theoretical_loss": 3.4119024579098225, + "tokens_seen": 2134835200 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035917171717171723, + "loss": 1.3673, + "theoretical_loss": 3.4118938367090736, + "tokens_seen": 2134900736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591515151515152, + "loss": 1.4432, + "theoretical_loss": 3.411885215847069, + "tokens_seen": 2134966272 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035913131313131314, + "loss": 1.4495, + "theoretical_loss": 3.4118765953237844, + "tokens_seen": 2135031808 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.43487292528152466, + "objective/train/avg_token_score": 0.015729127451777458, + "objective/train/docs_used": 1208575, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4166325330734253, + "objective/train/original_loss": 3.7783589363098145, + "objective/train/theoretical_loss": 3.4118679751391965, + "objective/train/tokens_used": 493956576, + "objective/train/unlikelihood_loss": 0.09019943326711655, + "theoretical_loss": 3.4118679751391965, + "tokens_seen": 2135097344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591111111111111, + "loss": 1.3516, + "theoretical_loss": 3.4118679751391965, + "tokens_seen": 2135097344 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035909090909090906, + "loss": 1.6024, + "theoretical_loss": 3.411859355293281, + "tokens_seen": 2135162880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590707070707071, + "loss": 1.5744, + "theoretical_loss": 3.4118507357860146, + "tokens_seen": 2135228416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590505050505051, + "loss": 1.2641, + "theoretical_loss": 3.4118421166173736, + "tokens_seen": 2135293952 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035903030303030304, + "loss": 1.4731, + "theoretical_loss": 3.4118334977873346, + "tokens_seen": 2135359488 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359010101010101, + "loss": 1.7547, + "theoretical_loss": 3.4118248792958736, + "tokens_seen": 2135425024 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358989898989899, + "loss": 1.5753, + "theoretical_loss": 3.4118162611429663, + "tokens_seen": 2135490560 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358969696969697, + "loss": 1.5385, + "theoretical_loss": 3.41180764332859, + "tokens_seen": 2135556096 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035894949494949497, + "loss": 1.6049, + "theoretical_loss": 3.41179902585272, + "tokens_seen": 2135621632 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035892929292929293, + "loss": 1.5225, + "theoretical_loss": 3.411790408715334, + "tokens_seen": 2135687168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589090909090909, + "loss": 1.5541, + "theoretical_loss": 3.411781791916407, + "tokens_seen": 2135752704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588888888888889, + "loss": 1.3487, + "theoretical_loss": 3.411773175455916, + "tokens_seen": 2135818240 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588686868686869, + "loss": 1.56, + "theoretical_loss": 3.4117645593338377, + "tokens_seen": 2135883776 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035884848484848486, + "loss": 1.4217, + "theoretical_loss": 3.411755943550147, + "tokens_seen": 2135949312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588282828282828, + "loss": 1.5536, + "theoretical_loss": 3.4117473281048216, + "tokens_seen": 2136014848 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035880808080808083, + "loss": 1.3469, + "theoretical_loss": 3.4117387129978374, + "tokens_seen": 2136080384 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587878787878788, + "loss": 1.5176, + "theoretical_loss": 3.4117300982291705, + "tokens_seen": 2136145920 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587676767676768, + "loss": 1.3404, + "theoretical_loss": 3.4117214837987975, + "tokens_seen": 2136211456 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035874747474747476, + "loss": 1.4812, + "theoretical_loss": 3.4117128697066947, + "tokens_seen": 2136276992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587272727272727, + "loss": 1.4882, + "theoretical_loss": 3.4117042559528388, + "tokens_seen": 2136342528 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587070707070707, + "loss": 1.6542, + "theoretical_loss": 3.411695642537205, + "tokens_seen": 2136408064 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586868686868687, + "loss": 1.5705, + "theoretical_loss": 3.411687029459771, + "tokens_seen": 2136473600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586666666666667, + "loss": 1.5797, + "theoretical_loss": 3.4116784167205125, + "tokens_seen": 2136539136 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035864646464646465, + "loss": 1.3391, + "theoretical_loss": 3.4116698043194056, + "tokens_seen": 2136604672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586262626262626, + "loss": 1.4513, + "theoretical_loss": 3.411661192256427, + "tokens_seen": 2136670208 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.5114858150482178, + "objective/train/avg_token_score": 0.026600955054163933, + "objective/train/docs_used": 1209997, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.747342824935913, + "objective/train/original_loss": 4.3426594734191895, + "objective/train/theoretical_loss": 3.4116525805315527, + "objective/train/tokens_used": 495594976, + "objective/train/unlikelihood_loss": 0.055273815989494324, + "theoretical_loss": 3.4116525805315527, + "tokens_seen": 2136735744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586060606060606, + "loss": 1.6058, + "theoretical_loss": 3.4116525805315527, + "tokens_seen": 2136735744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585858585858586, + "loss": 1.3531, + "theoretical_loss": 3.4116439691447598, + "tokens_seen": 2136801280 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585656565656566, + "loss": 1.5037, + "theoretical_loss": 3.4116353580960244, + "tokens_seen": 2136866816 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035854545454545454, + "loss": 1.4249, + "theoretical_loss": 3.411626747385322, + "tokens_seen": 2136932352 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035852525252525255, + "loss": 1.2405, + "theoretical_loss": 3.4116181370126304, + "tokens_seen": 2136997888 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585050505050505, + "loss": 1.2637, + "theoretical_loss": 3.4116095269779247, + "tokens_seen": 2137063424 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035848484848484847, + "loss": 1.3767, + "theoretical_loss": 3.4116009172811816, + "tokens_seen": 2137128960 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584646464646465, + "loss": 1.465, + "theoretical_loss": 3.411592307922378, + "tokens_seen": 2137194496 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035844444444444444, + "loss": 1.4217, + "theoretical_loss": 3.41158369890149, + "tokens_seen": 2137260032 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035842424242424245, + "loss": 1.7356, + "theoretical_loss": 3.411575090218494, + "tokens_seen": 2137325568 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584040404040404, + "loss": 1.3519, + "theoretical_loss": 3.4115664818733658, + "tokens_seen": 2137391104 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035838383838383836, + "loss": 1.399, + "theoretical_loss": 3.411557873866083, + "tokens_seen": 2137456640 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035836363636363637, + "loss": 1.4519, + "theoretical_loss": 3.4115492661966202, + "tokens_seen": 2137522176 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583434343434344, + "loss": 1.5747, + "theoretical_loss": 3.411540658864955, + "tokens_seen": 2137587712 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035832323232323234, + "loss": 1.3938, + "theoretical_loss": 3.411532051871064, + "tokens_seen": 2137653248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583030303030303, + "loss": 1.4424, + "theoretical_loss": 3.4115234452149235, + "tokens_seen": 2137718784 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035828282828282825, + "loss": 1.0837, + "theoretical_loss": 3.411514838896509, + "tokens_seen": 2137784320 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035826262626262626, + "loss": 1.4307, + "theoretical_loss": 3.411506232915798, + "tokens_seen": 2137849856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582424242424243, + "loss": 1.3948, + "theoretical_loss": 3.411497627272766, + "tokens_seen": 2137915392 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035822222222222223, + "loss": 1.3995, + "theoretical_loss": 3.41148902196739, + "tokens_seen": 2137980928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582020202020202, + "loss": 1.5618, + "theoretical_loss": 3.4114804169996464, + "tokens_seen": 2138046464 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035818181818181815, + "loss": 1.4169, + "theoretical_loss": 3.411471812369511, + "tokens_seen": 2138112000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581616161616162, + "loss": 1.6591, + "theoretical_loss": 3.4114632080769605, + "tokens_seen": 2138177536 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035814141414141417, + "loss": 1.5279, + "theoretical_loss": 3.4114546041219715, + "tokens_seen": 2138243072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581212121212121, + "loss": 1.2843, + "theoretical_loss": 3.4114460005045206, + "tokens_seen": 2138308608 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.43316227197647095, + "objective/train/avg_token_score": 0.01263100653886795, + "objective/train/docs_used": 1210838, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.406528353691101, + "objective/train/original_loss": 5.164313793182373, + "objective/train/theoretical_loss": 3.4114373972245837, + "objective/train/tokens_used": 497233376, + "objective/train/unlikelihood_loss": 0.04017988219857216, + "theoretical_loss": 3.4114373972245837, + "tokens_seen": 2138374144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581010101010101, + "loss": 1.2272, + "theoretical_loss": 3.4114373972245837, + "tokens_seen": 2138374144 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035808080808080804, + "loss": 1.497, + "theoretical_loss": 3.4114287942821377, + "tokens_seen": 2138439680 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580606060606061, + "loss": 1.4253, + "theoretical_loss": 3.4114201916771583, + "tokens_seen": 2138505216 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035804040404040406, + "loss": 1.4705, + "theoretical_loss": 3.4114115894096226, + "tokens_seen": 2138570752 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358020202020202, + "loss": 1.4205, + "theoretical_loss": 3.411402987479507, + "tokens_seen": 2138636288 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358, + "loss": 1.4689, + "theoretical_loss": 3.4113943858867875, + "tokens_seen": 2138701824 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357979797979798, + "loss": 1.4757, + "theoretical_loss": 3.411385784631441, + "tokens_seen": 2138767360 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357959595959596, + "loss": 1.5008, + "theoretical_loss": 3.4113771837134435, + "tokens_seen": 2138832896 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035793939393939395, + "loss": 1.3656, + "theoretical_loss": 3.411368583132772, + "tokens_seen": 2138898432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579191919191919, + "loss": 1.459, + "theoretical_loss": 3.411359982889402, + "tokens_seen": 2138963968 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035789898989898987, + "loss": 1.1593, + "theoretical_loss": 3.4113513829833106, + "tokens_seen": 2139029504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578787878787879, + "loss": 1.395, + "theoretical_loss": 3.411342783414474, + "tokens_seen": 2139095040 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578585858585859, + "loss": 1.3069, + "theoretical_loss": 3.411334184182869, + "tokens_seen": 2139160576 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035783838383838385, + "loss": 1.5065, + "theoretical_loss": 3.411325585288472, + "tokens_seen": 2139226112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578181818181818, + "loss": 1.4873, + "theoretical_loss": 3.411316986731259, + "tokens_seen": 2139291648 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577979797979798, + "loss": 1.5681, + "theoretical_loss": 3.411308388511207, + "tokens_seen": 2139357184 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035777777777777777, + "loss": 1.456, + "theoretical_loss": 3.4112997906282923, + "tokens_seen": 2139422720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577575757575758, + "loss": 1.5545, + "theoretical_loss": 3.4112911930824907, + "tokens_seen": 2139488256 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035773737373737374, + "loss": 1.448, + "theoretical_loss": 3.4112825958737796, + "tokens_seen": 2139553792 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577171717171717, + "loss": 1.5469, + "theoretical_loss": 3.411273999002135, + "tokens_seen": 2139619328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576969696969697, + "loss": 1.6172, + "theoretical_loss": 3.4112654024675333, + "tokens_seen": 2139684864 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576767676767677, + "loss": 1.4533, + "theoretical_loss": 3.411256806269951, + "tokens_seen": 2139750400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576565656565657, + "loss": 1.5875, + "theoretical_loss": 3.411248210409364, + "tokens_seen": 2139815936 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035763636363636363, + "loss": 1.3138, + "theoretical_loss": 3.4112396148857504, + "tokens_seen": 2139881472 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035761616161616164, + "loss": 1.3831, + "theoretical_loss": 3.411231019699086, + "tokens_seen": 2139947008 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.38550829887390137, + "objective/train/avg_token_score": 0.010996246710419655, + "objective/train/docs_used": 1212386, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3142551183700562, + "objective/train/original_loss": 4.66601037979126, + "objective/train/theoretical_loss": 3.411222424849346, + "objective/train/tokens_used": 498871776, + "objective/train/unlikelihood_loss": 0.042978379875421524, + "theoretical_loss": 3.411222424849346, + "tokens_seen": 2140012544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575959595959596, + "loss": 1.4413, + "theoretical_loss": 3.411222424849346, + "tokens_seen": 2140012544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575757575757576, + "loss": 1.4538, + "theoretical_loss": 3.411213830336508, + "tokens_seen": 2140078080 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035755555555555557, + "loss": 1.3922, + "theoretical_loss": 3.4112052361605487, + "tokens_seen": 2140143616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575353535353535, + "loss": 1.3603, + "theoretical_loss": 3.4111966423214435, + "tokens_seen": 2140209152 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035751515151515153, + "loss": 1.5785, + "theoretical_loss": 3.41118804881917, + "tokens_seen": 2140274688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574949494949495, + "loss": 1.6913, + "theoretical_loss": 3.4111794556537047, + "tokens_seen": 2140340224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574747474747475, + "loss": 1.4119, + "theoretical_loss": 3.411170862825023, + "tokens_seen": 2140405760 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035745454545454546, + "loss": 1.2982, + "theoretical_loss": 3.4111622703331017, + "tokens_seen": 2140471296 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035743434343434347, + "loss": 1.2424, + "theoretical_loss": 3.4111536781779184, + "tokens_seen": 2140536832 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035741414141414143, + "loss": 1.5727, + "theoretical_loss": 3.4111450863594484, + "tokens_seen": 2140602368 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573939393939394, + "loss": 1.649, + "theoretical_loss": 3.4111364948776686, + "tokens_seen": 2140667904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573737373737374, + "loss": 1.4136, + "theoretical_loss": 3.411127903732556, + "tokens_seen": 2140733440 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035735353535353535, + "loss": 1.4533, + "theoretical_loss": 3.411119312924086, + "tokens_seen": 2140798976 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035733333333333336, + "loss": 1.5858, + "theoretical_loss": 3.4111107224522357, + "tokens_seen": 2140864512 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573131313131313, + "loss": 1.4188, + "theoretical_loss": 3.411102132316982, + "tokens_seen": 2140930048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003572929292929293, + "loss": 1.2086, + "theoretical_loss": 3.411093542518301, + "tokens_seen": 2140995584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003572727272727273, + "loss": 1.5396, + "theoretical_loss": 3.411084953056169, + "tokens_seen": 2141061120 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003572525252525253, + "loss": 1.3101, + "theoretical_loss": 3.411076363930563, + "tokens_seen": 2141126656 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035723232323232326, + "loss": 1.448, + "theoretical_loss": 3.411067775141459, + "tokens_seen": 2141192192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003572121212121212, + "loss": 1.5575, + "theoretical_loss": 3.411059186688834, + "tokens_seen": 2141257728 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035719191919191917, + "loss": 1.4125, + "theoretical_loss": 3.411050598572664, + "tokens_seen": 2141323264 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571717171717172, + "loss": 1.8047, + "theoretical_loss": 3.4110420107929262, + "tokens_seen": 2141388800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571515151515152, + "loss": 1.4474, + "theoretical_loss": 3.4110334233495965, + "tokens_seen": 2141454336 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035713131313131315, + "loss": 1.6298, + "theoretical_loss": 3.4110248362426514, + "tokens_seen": 2141519872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571111111111111, + "loss": 1.6019, + "theoretical_loss": 3.4110162494720684, + "tokens_seen": 2141585408 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.3673631548881531, + "objective/train/avg_token_score": 0.01528299879282713, + "objective/train/docs_used": 1213144, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2732948064804077, + "objective/train/original_loss": 4.2406415939331055, + "objective/train/theoretical_loss": 3.4110076630378225, + "objective/train/tokens_used": 500510176, + "objective/train/unlikelihood_loss": 0.07076766341924667, + "theoretical_loss": 3.4110076630378225, + "tokens_seen": 2141650944 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035709090909090906, + "loss": 1.3349, + "theoretical_loss": 3.4110076630378225, + "tokens_seen": 2141650944 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035707070707070713, + "loss": 1.3803, + "theoretical_loss": 3.410999076939892, + "tokens_seen": 2141716480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570505050505051, + "loss": 1.4504, + "theoretical_loss": 3.410990491178252, + "tokens_seen": 2141782016 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035703030303030304, + "loss": 1.5414, + "theoretical_loss": 3.4109819057528794, + "tokens_seen": 2141847552 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357010101010101, + "loss": 1.4387, + "theoretical_loss": 3.4109733206637514, + "tokens_seen": 2141913088 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035698989898989896, + "loss": 1.3521, + "theoretical_loss": 3.4109647359108437, + "tokens_seen": 2141978624 + }, + { + "epoch": 0.29, + "learning_rate": 0.000356969696969697, + "loss": 1.3918, + "theoretical_loss": 3.4109561514941333, + "tokens_seen": 2142044160 + }, + { + "epoch": 0.29, + "learning_rate": 0.000356949494949495, + "loss": 1.4157, + "theoretical_loss": 3.4109475674135963, + "tokens_seen": 2142109696 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035692929292929293, + "loss": 1.3731, + "theoretical_loss": 3.41093898366921, + "tokens_seen": 2142175232 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569090909090909, + "loss": 1.732, + "theoretical_loss": 3.41093040026095, + "tokens_seen": 2142240768 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035688888888888885, + "loss": 1.3342, + "theoretical_loss": 3.410921817188794, + "tokens_seen": 2142306304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568686868686869, + "loss": 1.462, + "theoretical_loss": 3.4109132344527175, + "tokens_seen": 2142371840 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035684848484848487, + "loss": 1.5058, + "theoretical_loss": 3.4109046520526975, + "tokens_seen": 2142437376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568282828282828, + "loss": 1.4065, + "theoretical_loss": 3.410896069988711, + "tokens_seen": 2142502912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568080808080808, + "loss": 1.5842, + "theoretical_loss": 3.410887488260734, + "tokens_seen": 2142568448 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567878787878788, + "loss": 1.4824, + "theoretical_loss": 3.410878906868743, + "tokens_seen": 2142633984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567676767676768, + "loss": 1.3311, + "theoretical_loss": 3.4108703258127147, + "tokens_seen": 2142699520 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035674747474747476, + "loss": 1.405, + "theoretical_loss": 3.4108617450926255, + "tokens_seen": 2142765056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567272727272727, + "loss": 1.6685, + "theoretical_loss": 3.4108531647084526, + "tokens_seen": 2142830592 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567070707070707, + "loss": 1.2828, + "theoretical_loss": 3.4108445846601727, + "tokens_seen": 2142896128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003566868686868687, + "loss": 1.3849, + "theoretical_loss": 3.410836004947761, + "tokens_seen": 2142961664 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003566666666666667, + "loss": 1.2875, + "theoretical_loss": 3.4108274255711954, + "tokens_seen": 2143027200 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035664646464646466, + "loss": 1.4101, + "theoretical_loss": 3.410818846530452, + "tokens_seen": 2143092736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003566262626262626, + "loss": 1.4154, + "theoretical_loss": 3.4108102678255072, + "tokens_seen": 2143158272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003566060606060606, + "loss": 1.1729, + "theoretical_loss": 3.410801689456338, + "tokens_seen": 2143223808 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.34463587403297424, + "objective/train/avg_token_score": 0.016467271372675896, + "objective/train/docs_used": 1213818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0075488090515137, + "objective/train/original_loss": 4.430059909820557, + "objective/train/theoretical_loss": 3.4107931114229206, + "objective/train/tokens_used": 502148576, + "objective/train/unlikelihood_loss": 0.06894190609455109, + "theoretical_loss": 3.4107931114229206, + "tokens_seen": 2143289344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565858585858586, + "loss": 1.397, + "theoretical_loss": 3.4107931114229206, + "tokens_seen": 2143289344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565656565656566, + "loss": 1.3651, + "theoretical_loss": 3.410784533725232, + "tokens_seen": 2143354880 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035654545454545455, + "loss": 1.5144, + "theoretical_loss": 3.4107759563632487, + "tokens_seen": 2143420416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565252525252525, + "loss": 1.5995, + "theoretical_loss": 3.4107673793369466, + "tokens_seen": 2143485952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565050505050505, + "loss": 1.5493, + "theoretical_loss": 3.4107588026463036, + "tokens_seen": 2143551488 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564848484848485, + "loss": 1.3524, + "theoretical_loss": 3.410750226291295, + "tokens_seen": 2143617024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564646464646465, + "loss": 1.4136, + "theoretical_loss": 3.4107416502718984, + "tokens_seen": 2143682560 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035644444444444444, + "loss": 1.5132, + "theoretical_loss": 3.41073307458809, + "tokens_seen": 2143748096 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035642424242424245, + "loss": 1.421, + "theoretical_loss": 3.4107244992398464, + "tokens_seen": 2143813632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564040404040404, + "loss": 1.6114, + "theoretical_loss": 3.4107159242271443, + "tokens_seen": 2143879168 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035638383838383837, + "loss": 1.5623, + "theoretical_loss": 3.41070734954996, + "tokens_seen": 2143944704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563636363636364, + "loss": 1.5388, + "theoretical_loss": 3.41069877520827, + "tokens_seen": 2144010240 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035634343434343433, + "loss": 1.2676, + "theoretical_loss": 3.410690201202052, + "tokens_seen": 2144075776 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035632323232323234, + "loss": 1.4555, + "theoretical_loss": 3.4106816275312815, + "tokens_seen": 2144141312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563030303030303, + "loss": 1.322, + "theoretical_loss": 3.410673054195936, + "tokens_seen": 2144206848 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035628282828282826, + "loss": 1.5607, + "theoretical_loss": 3.410664481195991, + "tokens_seen": 2144272384 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035626262626262627, + "loss": 1.4592, + "theoretical_loss": 3.4106559085314236, + "tokens_seen": 2144337920 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562424242424243, + "loss": 1.5127, + "theoretical_loss": 3.4106473362022114, + "tokens_seen": 2144403456 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035622222222222224, + "loss": 1.4354, + "theoretical_loss": 3.41063876420833, + "tokens_seen": 2144468992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562020202020202, + "loss": 1.6413, + "theoretical_loss": 3.410630192549756, + "tokens_seen": 2144534528 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035618181818181815, + "loss": 1.6882, + "theoretical_loss": 3.4106216212264666, + "tokens_seen": 2144600064 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035616161616161616, + "loss": 1.4265, + "theoretical_loss": 3.4106130502384375, + "tokens_seen": 2144665600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003561414141414142, + "loss": 1.3411, + "theoretical_loss": 3.410604479585647, + "tokens_seen": 2144731136 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035612121212121213, + "loss": 1.2964, + "theoretical_loss": 3.4105959092680695, + "tokens_seen": 2144796672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003561010101010101, + "loss": 1.3232, + "theoretical_loss": 3.410587339285684, + "tokens_seen": 2144862208 + }, + { + "epoch": 0.29, + "objective/train/aligned_token_count": 0.3159213066101074, + "objective/train/avg_token_score": 0.012928572483360767, + "objective/train/docs_used": 1215022, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9320014715194702, + "objective/train/original_loss": 4.363121032714844, + "objective/train/theoretical_loss": 3.4105787696384655, + "objective/train/tokens_used": 503786976, + "objective/train/unlikelihood_loss": 0.05617682263255119, + "theoretical_loss": 3.4105787696384655, + "tokens_seen": 2144927744 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035608080808080804, + "loss": 1.3469, + "theoretical_loss": 3.4105787696384655, + "tokens_seen": 2144927744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003560606060606061, + "loss": 1.4266, + "theoretical_loss": 3.4105702003263914, + "tokens_seen": 2144993280 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035604040404040407, + "loss": 1.4661, + "theoretical_loss": 3.410561631349438, + "tokens_seen": 2145058816 + }, + { + "epoch": 0.3, + "learning_rate": 0.000356020202020202, + "loss": 1.4141, + "theoretical_loss": 3.410553062707582, + "tokens_seen": 2145124352 + }, + { + "epoch": 0.3, + "learning_rate": 0.000356, + "loss": 1.3925, + "theoretical_loss": 3.4105444944008, + "tokens_seen": 2145189888 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355979797979798, + "loss": 1.3971, + "theoretical_loss": 3.4105359264290693, + "tokens_seen": 2145255424 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355959595959596, + "loss": 1.5648, + "theoretical_loss": 3.4105273587923657, + "tokens_seen": 2145320960 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035593939393939396, + "loss": 1.5217, + "theoretical_loss": 3.4105187914906665, + "tokens_seen": 2145386496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559191919191919, + "loss": 1.3943, + "theoretical_loss": 3.4105102245239483, + "tokens_seen": 2145452032 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035589898989898987, + "loss": 1.2379, + "theoretical_loss": 3.410501657892187, + "tokens_seen": 2145517568 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035587878787878794, + "loss": 1.6016, + "theoretical_loss": 3.4104930915953604, + "tokens_seen": 2145583104 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558585858585859, + "loss": 1.1341, + "theoretical_loss": 3.410484525633444, + "tokens_seen": 2145648640 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035583838383838385, + "loss": 1.6203, + "theoretical_loss": 3.4104759600064156, + "tokens_seen": 2145714176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558181818181818, + "loss": 1.5268, + "theoretical_loss": 3.410467394714251, + "tokens_seen": 2145779712 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035579797979797977, + "loss": 1.193, + "theoretical_loss": 3.4104588297569283, + "tokens_seen": 2145845248 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035577777777777783, + "loss": 1.3764, + "theoretical_loss": 3.410450265134422, + "tokens_seen": 2145910784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557575757575758, + "loss": 1.3695, + "theoretical_loss": 3.41044170084671, + "tokens_seen": 2145976320 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035573737373737374, + "loss": 1.3551, + "theoretical_loss": 3.4104331368937695, + "tokens_seen": 2146041856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557171717171717, + "loss": 1.6121, + "theoretical_loss": 3.4104245732755762, + "tokens_seen": 2146107392 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003556969696969697, + "loss": 1.4464, + "theoretical_loss": 3.4104160099921077, + "tokens_seen": 2146172928 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003556767676767677, + "loss": 1.4414, + "theoretical_loss": 3.41040744704334, + "tokens_seen": 2146238464 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003556565656565657, + "loss": 1.4752, + "theoretical_loss": 3.4103988844292497, + "tokens_seen": 2146304000 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035563636363636364, + "loss": 1.4039, + "theoretical_loss": 3.410390322149814, + "tokens_seen": 2146369536 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003556161616161616, + "loss": 1.3964, + "theoretical_loss": 3.4103817602050093, + "tokens_seen": 2146435072 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555959595959596, + "loss": 1.6103, + "theoretical_loss": 3.4103731985948125, + "tokens_seen": 2146500608 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.31359970569610596, + "objective/train/avg_token_score": 0.02061370201408863, + "objective/train/docs_used": 1215651, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.065711259841919, + "objective/train/original_loss": 4.7896342277526855, + "objective/train/theoretical_loss": 3.4103646373192005, + "objective/train/tokens_used": 505425376, + "objective/train/unlikelihood_loss": 0.05333314463496208, + "theoretical_loss": 3.4103646373192005, + "tokens_seen": 2146566144 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555757575757576, + "loss": 1.3544, + "theoretical_loss": 3.4103646373192005, + "tokens_seen": 2146566144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035555555555555557, + "loss": 1.3426, + "theoretical_loss": 3.4103560763781493, + "tokens_seen": 2146631680 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035553535353535353, + "loss": 1.3554, + "theoretical_loss": 3.4103475157716363, + "tokens_seen": 2146697216 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035551515151515154, + "loss": 1.7167, + "theoretical_loss": 3.410338955499638, + "tokens_seen": 2146762752 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554949494949495, + "loss": 1.5226, + "theoretical_loss": 3.4103303955621307, + "tokens_seen": 2146828288 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554747474747475, + "loss": 1.6862, + "theoretical_loss": 3.410321835959092, + "tokens_seen": 2146893824 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035545454545454547, + "loss": 1.5275, + "theoretical_loss": 3.4103132766904976, + "tokens_seen": 2146959360 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554343434343434, + "loss": 1.1603, + "theoretical_loss": 3.4103047177563255, + "tokens_seen": 2147024896 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035541414141414143, + "loss": 1.5835, + "theoretical_loss": 3.410296159156551, + "tokens_seen": 2147090432 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553939393939394, + "loss": 1.4474, + "theoretical_loss": 3.410287600891152, + "tokens_seen": 2147155968 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553737373737374, + "loss": 1.6101, + "theoretical_loss": 3.410279042960104, + "tokens_seen": 2147221504 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035535353535353536, + "loss": 1.4196, + "theoretical_loss": 3.4102704853633847, + "tokens_seen": 2147287040 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553333333333333, + "loss": 1.2991, + "theoretical_loss": 3.4102619281009714, + "tokens_seen": 2147352576 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553131313131313, + "loss": 1.4851, + "theoretical_loss": 3.410253371172839, + "tokens_seen": 2147418112 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552929292929293, + "loss": 1.5103, + "theoretical_loss": 3.4102448145789657, + "tokens_seen": 2147483648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552727272727273, + "loss": 1.4754, + "theoretical_loss": 3.410236258319328, + "tokens_seen": 2147549184 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035525252525252525, + "loss": 1.3945, + "theoretical_loss": 3.4102277023939025, + "tokens_seen": 2147614720 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035523232323232326, + "loss": 1.3597, + "theoretical_loss": 3.4102191468026657, + "tokens_seen": 2147680256 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552121212121212, + "loss": 1.6051, + "theoretical_loss": 3.4102105915455945, + "tokens_seen": 2147745792 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551919191919192, + "loss": 1.4595, + "theoretical_loss": 3.4102020366226657, + "tokens_seen": 2147811328 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551717171717172, + "loss": 1.4641, + "theoretical_loss": 3.410193482033856, + "tokens_seen": 2147876864 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035515151515151514, + "loss": 1.4089, + "theoretical_loss": 3.4101849277791425, + "tokens_seen": 2147942400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035513131313131315, + "loss": 1.2692, + "theoretical_loss": 3.4101763738585014, + "tokens_seen": 2148007936 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551111111111111, + "loss": 1.3462, + "theoretical_loss": 3.4101678202719103, + "tokens_seen": 2148073472 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035509090909090907, + "loss": 1.634, + "theoretical_loss": 3.4101592670193446, + "tokens_seen": 2148139008 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.45839443802833557, + "objective/train/avg_token_score": 0.009575318545103073, + "objective/train/docs_used": 1216251, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5462843179702759, + "objective/train/original_loss": 4.091464996337891, + "objective/train/theoretical_loss": 3.4101507141007827, + "objective/train/tokens_used": 507063776, + "objective/train/unlikelihood_loss": 0.06715980917215347, + "theoretical_loss": 3.4101507141007827, + "tokens_seen": 2148204544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550707070707071, + "loss": 1.4466, + "theoretical_loss": 3.4101507141007827, + "tokens_seen": 2148204544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550505050505051, + "loss": 1.5371, + "theoretical_loss": 3.4101421615162, + "tokens_seen": 2148270080 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035503030303030305, + "loss": 1.6062, + "theoretical_loss": 3.410133609265574, + "tokens_seen": 2148335616 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355010101010101, + "loss": 1.2978, + "theoretical_loss": 3.410125057348881, + "tokens_seen": 2148401152 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035498989898989896, + "loss": 1.3665, + "theoretical_loss": 3.4101165057660987, + "tokens_seen": 2148466688 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035496969696969697, + "loss": 1.4329, + "theoretical_loss": 3.4101079545172026, + "tokens_seen": 2148532224 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354949494949495, + "loss": 1.4802, + "theoretical_loss": 3.41009940360217, + "tokens_seen": 2148597760 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035492929292929294, + "loss": 1.3216, + "theoretical_loss": 3.4100908530209786, + "tokens_seen": 2148663296 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549090909090909, + "loss": 1.5478, + "theoretical_loss": 3.4100823027736036, + "tokens_seen": 2148728832 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035488888888888885, + "loss": 1.4956, + "theoretical_loss": 3.410073752860023, + "tokens_seen": 2148794368 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548686868686869, + "loss": 1.5093, + "theoretical_loss": 3.4100652032802135, + "tokens_seen": 2148859904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548484848484849, + "loss": 1.1582, + "theoretical_loss": 3.410056654034151, + "tokens_seen": 2148925440 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035482828282828283, + "loss": 1.5054, + "theoretical_loss": 3.4100481051218132, + "tokens_seen": 2148990976 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548080808080808, + "loss": 1.2743, + "theoretical_loss": 3.4100395565431763, + "tokens_seen": 2149056512 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035478787878787875, + "loss": 1.4463, + "theoretical_loss": 3.4100310082982173, + "tokens_seen": 2149122048 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003547676767676768, + "loss": 1.2762, + "theoretical_loss": 3.410022460386913, + "tokens_seen": 2149187584 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035474747474747477, + "loss": 1.3302, + "theoretical_loss": 3.4100139128092404, + "tokens_seen": 2149253120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003547272727272727, + "loss": 1.3001, + "theoretical_loss": 3.4100053655651763, + "tokens_seen": 2149318656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003547070707070707, + "loss": 1.5577, + "theoretical_loss": 3.409996818654697, + "tokens_seen": 2149384192 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546868686868687, + "loss": 1.2636, + "theoretical_loss": 3.4099882720777797, + "tokens_seen": 2149449728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546666666666667, + "loss": 1.2444, + "theoretical_loss": 3.4099797258344013, + "tokens_seen": 2149515264 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035464646464646466, + "loss": 1.1734, + "theoretical_loss": 3.4099711799245385, + "tokens_seen": 2149580800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546262626262626, + "loss": 1.2099, + "theoretical_loss": 3.4099626343481684, + "tokens_seen": 2149646336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546060606060606, + "loss": 1.3326, + "theoretical_loss": 3.409954089105267, + "tokens_seen": 2149711872 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545858585858586, + "loss": 1.5443, + "theoretical_loss": 3.4099455441958124, + "tokens_seen": 2149777408 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.3271627426147461, + "objective/train/avg_token_score": 0.021921349689364433, + "objective/train/docs_used": 1217415, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.14596688747406, + "objective/train/original_loss": 4.5187530517578125, + "objective/train/theoretical_loss": 3.40993699961978, + "objective/train/tokens_used": 508702176, + "objective/train/unlikelihood_loss": 0.05059804767370224, + "theoretical_loss": 3.40993699961978, + "tokens_seen": 2149842944 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545656565656566, + "loss": 1.4055, + "theoretical_loss": 3.40993699961978, + "tokens_seen": 2149842944 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035454545454545455, + "loss": 1.3152, + "theoretical_loss": 3.4099284553771474, + "tokens_seen": 2149908480 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545252525252525, + "loss": 1.3024, + "theoretical_loss": 3.4099199114678918, + "tokens_seen": 2149974016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545050505050505, + "loss": 1.5076, + "theoretical_loss": 3.409911367891989, + "tokens_seen": 2150039552 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544848484848485, + "loss": 1.4309, + "theoretical_loss": 3.4099028246494165, + "tokens_seen": 2150105088 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544646464646465, + "loss": 1.2262, + "theoretical_loss": 3.409894281740151, + "tokens_seen": 2150170624 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035444444444444445, + "loss": 1.4737, + "theoretical_loss": 3.40988573916417, + "tokens_seen": 2150236160 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544242424242424, + "loss": 1.424, + "theoretical_loss": 3.4098771969214487, + "tokens_seen": 2150301696 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544040404040404, + "loss": 1.4056, + "theoretical_loss": 3.409868655011966, + "tokens_seen": 2150367232 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035438383838383837, + "loss": 1.4112, + "theoretical_loss": 3.4098601134356965, + "tokens_seen": 2150432768 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543636363636364, + "loss": 1.5308, + "theoretical_loss": 3.4098515721926193, + "tokens_seen": 2150498304 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035434343434343434, + "loss": 1.3105, + "theoretical_loss": 3.4098430312827097, + "tokens_seen": 2150563840 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035432323232323235, + "loss": 1.4338, + "theoretical_loss": 3.4098344907059452, + "tokens_seen": 2150629376 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543030303030303, + "loss": 1.3089, + "theoretical_loss": 3.4098259504623023, + "tokens_seen": 2150694912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542828282828283, + "loss": 1.2264, + "theoretical_loss": 3.4098174105517582, + "tokens_seen": 2150760448 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542626262626263, + "loss": 1.1657, + "theoretical_loss": 3.40980887097429, + "tokens_seen": 2150825984 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035424242424242423, + "loss": 1.5638, + "theoretical_loss": 3.4098003317298735, + "tokens_seen": 2150891520 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035422222222222224, + "loss": 1.4518, + "theoretical_loss": 3.4097917928184867, + "tokens_seen": 2150957056 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542020202020202, + "loss": 1.2664, + "theoretical_loss": 3.4097832542401063, + "tokens_seen": 2151022592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541818181818182, + "loss": 1.4255, + "theoretical_loss": 3.4097747159947085, + "tokens_seen": 2151088128 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035416161616161617, + "loss": 1.3902, + "theoretical_loss": 3.40976617808227, + "tokens_seen": 2151153664 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541414141414142, + "loss": 1.3423, + "theoretical_loss": 3.409757640502769, + "tokens_seen": 2151219200 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035412121212121214, + "loss": 1.5539, + "theoretical_loss": 3.409749103256181, + "tokens_seen": 2151284736 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541010101010101, + "loss": 1.4584, + "theoretical_loss": 3.4097405663424842, + "tokens_seen": 2151350272 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540808080808081, + "loss": 1.3504, + "theoretical_loss": 3.4097320297616545, + "tokens_seen": 2151415808 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.5538856387138367, + "objective/train/avg_token_score": 0.012418524362146854, + "objective/train/docs_used": 1217907, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7489923238754272, + "objective/train/original_loss": 3.712601661682129, + "objective/train/theoretical_loss": 3.4097234935136687, + "objective/train/tokens_used": 510340576, + "objective/train/unlikelihood_loss": 0.07619179040193558, + "theoretical_loss": 3.4097234935136687, + "tokens_seen": 2151481344 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035406060606060606, + "loss": 1.4716, + "theoretical_loss": 3.4097234935136687, + "tokens_seen": 2151481344 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035404040404040407, + "loss": 1.369, + "theoretical_loss": 3.4097149575985037, + "tokens_seen": 2151546880 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035402020202020203, + "loss": 1.4984, + "theoretical_loss": 3.4097064220161375, + "tokens_seen": 2151612416 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354, + "loss": 1.4327, + "theoretical_loss": 3.409697886766546, + "tokens_seen": 2151677952 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353979797979798, + "loss": 1.3699, + "theoretical_loss": 3.409689351849706, + "tokens_seen": 2151743488 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353959595959596, + "loss": 1.164, + "theoretical_loss": 3.409680817265595, + "tokens_seen": 2151809024 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035393939393939396, + "loss": 1.3928, + "theoretical_loss": 3.4096722830141895, + "tokens_seen": 2151874560 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003539191919191919, + "loss": 1.5867, + "theoretical_loss": 3.409663749095466, + "tokens_seen": 2151940096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538989898989899, + "loss": 1.1484, + "theoretical_loss": 3.4096552155094026, + "tokens_seen": 2152005632 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538787878787879, + "loss": 1.285, + "theoretical_loss": 3.409646682255975, + "tokens_seen": 2152071168 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538585858585859, + "loss": 1.5314, + "theoretical_loss": 3.409638149335161, + "tokens_seen": 2152136704 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035383838383838386, + "loss": 1.3147, + "theoretical_loss": 3.4096296167469364, + "tokens_seen": 2152202240 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538181818181818, + "loss": 1.4144, + "theoretical_loss": 3.4096210844912793, + "tokens_seen": 2152267776 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035379797979797977, + "loss": 1.151, + "theoretical_loss": 3.409612552568166, + "tokens_seen": 2152333312 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035377777777777784, + "loss": 1.1738, + "theoretical_loss": 3.4096040209775733, + "tokens_seen": 2152398848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537575757575758, + "loss": 1.3645, + "theoretical_loss": 3.4095954897194787, + "tokens_seen": 2152464384 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035373737373737375, + "loss": 1.4558, + "theoretical_loss": 3.4095869587938585, + "tokens_seen": 2152529920 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537171717171717, + "loss": 1.2696, + "theoretical_loss": 3.40957842820069, + "tokens_seen": 2152595456 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035369696969696966, + "loss": 1.3487, + "theoretical_loss": 3.4095698979399502, + "tokens_seen": 2152660992 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035367676767676773, + "loss": 1.449, + "theoretical_loss": 3.4095613680116155, + "tokens_seen": 2152726528 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536565656565657, + "loss": 1.3081, + "theoretical_loss": 3.4095528384156633, + "tokens_seen": 2152792064 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035363636363636364, + "loss": 1.3938, + "theoretical_loss": 3.4095443091520705, + "tokens_seen": 2152857600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536161616161616, + "loss": 1.3076, + "theoretical_loss": 3.4095357802208133, + "tokens_seen": 2152923136 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035359595959595956, + "loss": 1.2976, + "theoretical_loss": 3.4095272516218698, + "tokens_seen": 2152988672 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535757575757576, + "loss": 1.324, + "theoretical_loss": 3.4095187233552164, + "tokens_seen": 2153054208 + }, + { + "debugging/Self-BLEU-5": 0.4342780019633275, + "debugging/distinct-1-grams": 0.8078002938067941, + "debugging/distinct-2-grams": 0.9689768537714897, + "debugging/entropy-1-grams": 5.9515082084159445, + "debugging/entropy-2-grams": 6.74050317919558, + "debugging/length": 469.15384615384613, + "debugging/num_segments": 13, + "epoch": 0.3, + "objective/train/aligned_token_count": 0.574474573135376, + "objective/train/avg_token_score": 0.01960972137749195, + "objective/train/docs_used": 1219413, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7889963388442993, + "objective/train/original_loss": 3.6546130180358887, + "objective/train/theoretical_loss": 3.40951019542083, + "objective/train/tokens_used": 511978976, + "objective/train/unlikelihood_loss": 0.05853184312582016, + "theoretical_loss": 3.40951019542083, + "tokens_seen": 2153119744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535555555555556, + "loss": 1.5971, + "theoretical_loss": 3.40951019542083, + "tokens_seen": 2153119744 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035353535353535354, + "loss": 1.6635, + "theoretical_loss": 3.409501667818687, + "tokens_seen": 2153185280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535151515151515, + "loss": 1.7406, + "theoretical_loss": 3.409493140548766, + "tokens_seen": 2153250816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534949494949495, + "loss": 1.2769, + "theoretical_loss": 3.409484613611042, + "tokens_seen": 2153316352 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534747474747475, + "loss": 1.1519, + "theoretical_loss": 3.409476087005493, + "tokens_seen": 2153381888 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035345454545454547, + "loss": 1.4715, + "theoretical_loss": 3.4094675607320957, + "tokens_seen": 2153447424 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035343434343434343, + "loss": 1.6237, + "theoretical_loss": 3.4094590347908276, + "tokens_seen": 2153512960 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534141414141414, + "loss": 1.4639, + "theoretical_loss": 3.4094505091816645, + "tokens_seen": 2153578496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533939393939394, + "loss": 1.6356, + "theoretical_loss": 3.4094419839045846, + "tokens_seen": 2153644032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533737373737374, + "loss": 1.5859, + "theoretical_loss": 3.409433458959564, + "tokens_seen": 2153709568 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035335353535353536, + "loss": 1.1783, + "theoretical_loss": 3.40942493434658, + "tokens_seen": 2153775104 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533333333333333, + "loss": 1.5161, + "theoretical_loss": 3.4094164100656092, + "tokens_seen": 2153840640 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035331313131313133, + "loss": 1.3464, + "theoretical_loss": 3.4094078861166297, + "tokens_seen": 2153906176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532929292929293, + "loss": 1.1118, + "theoretical_loss": 3.4093993624996166, + "tokens_seen": 2153971712 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532727272727273, + "loss": 1.3196, + "theoretical_loss": 3.4093908392145487, + "tokens_seen": 2154037248 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035325252525252526, + "loss": 1.4936, + "theoretical_loss": 3.409382316261402, + "tokens_seen": 2154102784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532323232323232, + "loss": 1.6132, + "theoretical_loss": 3.4093737936401536, + "tokens_seen": 2154168320 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532121212121212, + "loss": 1.5541, + "theoretical_loss": 3.4093652713507803, + "tokens_seen": 2154233856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531919191919192, + "loss": 1.3592, + "theoretical_loss": 3.4093567493932597, + "tokens_seen": 2154299392 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531717171717172, + "loss": 1.5112, + "theoretical_loss": 3.4093482277675684, + "tokens_seen": 2154364928 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035315151515151515, + "loss": 1.5003, + "theoretical_loss": 3.4093397064736832, + "tokens_seen": 2154430464 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035313131313131316, + "loss": 1.6506, + "theoretical_loss": 3.4093311855115815, + "tokens_seen": 2154496000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531111111111111, + "loss": 1.5629, + "theoretical_loss": 3.4093226648812403, + "tokens_seen": 2154561536 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530909090909091, + "loss": 1.4242, + "theoretical_loss": 3.409314144582636, + "tokens_seen": 2154627072 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530707070707071, + "loss": 1.3268, + "theoretical_loss": 3.4093056246157456, + "tokens_seen": 2154692608 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.2871456444263458, + "objective/train/avg_token_score": 0.014688773080706596, + "objective/train/docs_used": 1220074, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.890113115310669, + "objective/train/original_loss": 5.298357963562012, + "objective/train/theoretical_loss": 3.4092971049805474, + "objective/train/tokens_used": 513617376, + "objective/train/unlikelihood_loss": 0.041902169585227966, + "theoretical_loss": 3.4092971049805474, + "tokens_seen": 2154758144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035305050505050504, + "loss": 1.1781, + "theoretical_loss": 3.4092971049805474, + "tokens_seen": 2154758144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035303030303030305, + "loss": 1.5121, + "theoretical_loss": 3.4092885856770168, + "tokens_seen": 2154823680 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353010101010101, + "loss": 1.4291, + "theoretical_loss": 3.4092800667051315, + "tokens_seen": 2154889216 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035298989898989897, + "loss": 1.4243, + "theoretical_loss": 3.4092715480648685, + "tokens_seen": 2154954752 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352969696969697, + "loss": 1.6243, + "theoretical_loss": 3.4092630297562048, + "tokens_seen": 2155020288 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352949494949495, + "loss": 1.7129, + "theoretical_loss": 3.4092545117791175, + "tokens_seen": 2155085824 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035292929292929295, + "loss": 1.5722, + "theoretical_loss": 3.4092459941335838, + "tokens_seen": 2155151360 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003529090909090909, + "loss": 1.3569, + "theoretical_loss": 3.4092374768195794, + "tokens_seen": 2155216896 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035288888888888886, + "loss": 1.4478, + "theoretical_loss": 3.4092289598370833, + "tokens_seen": 2155282432 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035286868686868687, + "loss": 1.3799, + "theoretical_loss": 3.4092204431860713, + "tokens_seen": 2155347968 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003528484848484849, + "loss": 1.3461, + "theoretical_loss": 3.4092119268665204, + "tokens_seen": 2155413504 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035282828282828284, + "loss": 1.4725, + "theoretical_loss": 3.409203410878408, + "tokens_seen": 2155479040 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003528080808080808, + "loss": 1.4399, + "theoretical_loss": 3.4091948952217113, + "tokens_seen": 2155544576 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035278787878787875, + "loss": 1.1607, + "theoretical_loss": 3.4091863798964064, + "tokens_seen": 2155610112 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527676767676768, + "loss": 1.6022, + "theoretical_loss": 3.409177864902471, + "tokens_seen": 2155675648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527474747474748, + "loss": 1.5076, + "theoretical_loss": 3.4091693502398828, + "tokens_seen": 2155741184 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035272727272727273, + "loss": 1.4086, + "theoretical_loss": 3.409160835908618, + "tokens_seen": 2155806720 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527070707070707, + "loss": 1.3605, + "theoretical_loss": 3.4091523219086532, + "tokens_seen": 2155872256 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035268686868686864, + "loss": 1.5611, + "theoretical_loss": 3.4091438082399668, + "tokens_seen": 2155937792 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526666666666667, + "loss": 1.4342, + "theoretical_loss": 3.4091352949025344, + "tokens_seen": 2156003328 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035264646464646467, + "loss": 1.239, + "theoretical_loss": 3.4091267818963336, + "tokens_seen": 2156068864 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526262626262626, + "loss": 1.3897, + "theoretical_loss": 3.4091182692213415, + "tokens_seen": 2156134400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526060606060606, + "loss": 1.5341, + "theoretical_loss": 3.4091097568775357, + "tokens_seen": 2156199936 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035258585858585865, + "loss": 1.328, + "theoretical_loss": 3.4091012448648925, + "tokens_seen": 2156265472 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525656565656566, + "loss": 1.2071, + "theoretical_loss": 3.409092733183389, + "tokens_seen": 2156331008 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.5043988227844238, + "objective/train/avg_token_score": 0.01132004614919424, + "objective/train/docs_used": 1221515, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7465596199035645, + "objective/train/original_loss": 4.144891738891602, + "objective/train/theoretical_loss": 3.4090842218330026, + "objective/train/tokens_used": 515255776, + "objective/train/unlikelihood_loss": 0.058709658682346344, + "theoretical_loss": 3.4090842218330026, + "tokens_seen": 2156396544 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035254545454545456, + "loss": 1.577, + "theoretical_loss": 3.4090842218330026, + "tokens_seen": 2156396544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525252525252525, + "loss": 1.4144, + "theoretical_loss": 3.40907571081371, + "tokens_seen": 2156462080 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525050505050505, + "loss": 1.622, + "theoretical_loss": 3.409067200125489, + "tokens_seen": 2156527616 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035248484848484854, + "loss": 1.4798, + "theoretical_loss": 3.4090586897683153, + "tokens_seen": 2156593152 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524646464646465, + "loss": 1.3924, + "theoretical_loss": 3.4090501797421675, + "tokens_seen": 2156658688 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035244444444444445, + "loss": 1.4228, + "theoretical_loss": 3.4090416700470216, + "tokens_seen": 2156724224 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524242424242424, + "loss": 1.3212, + "theoretical_loss": 3.409033160682855, + "tokens_seen": 2156789760 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524040404040404, + "loss": 1.2613, + "theoretical_loss": 3.409024651649645, + "tokens_seen": 2156855296 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035238383838383843, + "loss": 1.3956, + "theoretical_loss": 3.409016142947368, + "tokens_seen": 2156920832 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523636363636364, + "loss": 1.3073, + "theoretical_loss": 3.409007634576002, + "tokens_seen": 2156986368 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035234343434343434, + "loss": 1.3321, + "theoretical_loss": 3.408999126535523, + "tokens_seen": 2157051904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523232323232323, + "loss": 1.5444, + "theoretical_loss": 3.4089906188259094, + "tokens_seen": 2157117440 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523030303030303, + "loss": 1.4442, + "theoretical_loss": 3.408982111447137, + "tokens_seen": 2157182976 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522828282828283, + "loss": 1.347, + "theoretical_loss": 3.4089736043991836, + "tokens_seen": 2157248512 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522626262626263, + "loss": 1.5098, + "theoretical_loss": 3.4089650976820263, + "tokens_seen": 2157314048 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035224242424242424, + "loss": 1.5813, + "theoretical_loss": 3.4089565912956417, + "tokens_seen": 2157379584 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035222222222222225, + "loss": 1.3246, + "theoretical_loss": 3.4089480852400076, + "tokens_seen": 2157445120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522020202020202, + "loss": 1.2625, + "theoretical_loss": 3.408939579515101, + "tokens_seen": 2157510656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521818181818182, + "loss": 1.407, + "theoretical_loss": 3.408931074120898, + "tokens_seen": 2157576192 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521616161616162, + "loss": 1.4867, + "theoretical_loss": 3.4089225690573763, + "tokens_seen": 2157641728 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035214141414141413, + "loss": 1.3851, + "theoretical_loss": 3.4089140643245135, + "tokens_seen": 2157707264 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035212121212121214, + "loss": 1.5038, + "theoretical_loss": 3.408905559922286, + "tokens_seen": 2157772800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521010101010101, + "loss": 1.292, + "theoretical_loss": 3.4088970558506717, + "tokens_seen": 2157838336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003520808080808081, + "loss": 1.359, + "theoretical_loss": 3.408888552109647, + "tokens_seen": 2157903872 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035206060606060607, + "loss": 1.4124, + "theoretical_loss": 3.408880048699189, + "tokens_seen": 2157969408 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.4323069453239441, + "objective/train/avg_token_score": 0.02164054848253727, + "objective/train/docs_used": 1222265, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3648401498794556, + "objective/train/original_loss": 4.0131659507751465, + "objective/train/theoretical_loss": 3.4088715456192755, + "objective/train/tokens_used": 516894176, + "objective/train/unlikelihood_loss": 0.05729663372039795, + "theoretical_loss": 3.4088715456192755, + "tokens_seen": 2158034944 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352040404040404, + "loss": 1.4769, + "theoretical_loss": 3.4088715456192755, + "tokens_seen": 2158034944 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035202020202020203, + "loss": 1.4269, + "theoretical_loss": 3.4088630428698825, + "tokens_seen": 2158100480 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352, + "loss": 1.4243, + "theoretical_loss": 3.4088545404509882, + "tokens_seen": 2158166016 + }, + { + "epoch": 0.3, + "learning_rate": 0.000351979797979798, + "loss": 1.3082, + "theoretical_loss": 3.408846038362569, + "tokens_seen": 2158231552 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035195959595959596, + "loss": 1.6036, + "theoretical_loss": 3.4088375366046026, + "tokens_seen": 2158297088 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035193939393939397, + "loss": 1.5378, + "theoretical_loss": 3.4088290351770656, + "tokens_seen": 2158362624 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003519191919191919, + "loss": 1.5653, + "theoretical_loss": 3.4088205340799353, + "tokens_seen": 2158428160 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518989898989899, + "loss": 1.6522, + "theoretical_loss": 3.4088120333131893, + "tokens_seen": 2158493696 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518787878787879, + "loss": 1.4333, + "theoretical_loss": 3.408803532876804, + "tokens_seen": 2158559232 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035185858585858585, + "loss": 1.5149, + "theoretical_loss": 3.408795032770757, + "tokens_seen": 2158624768 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035183838383838386, + "loss": 1.4609, + "theoretical_loss": 3.408786532995025, + "tokens_seen": 2158690304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518181818181818, + "loss": 1.4859, + "theoretical_loss": 3.408778033549585, + "tokens_seen": 2158755840 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517979797979798, + "loss": 1.2978, + "theoretical_loss": 3.4087695344344153, + "tokens_seen": 2158821376 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517777777777778, + "loss": 1.3379, + "theoretical_loss": 3.4087610356494924, + "tokens_seen": 2158886912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517575757575758, + "loss": 1.4553, + "theoretical_loss": 3.408752537194793, + "tokens_seen": 2158952448 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035173737373737376, + "loss": 1.164, + "theoretical_loss": 3.408744039070294, + "tokens_seen": 2159017984 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517171717171717, + "loss": 1.5104, + "theoretical_loss": 3.408735541275974, + "tokens_seen": 2159083520 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035169696969696967, + "loss": 1.7157, + "theoretical_loss": 3.408727043811809, + "tokens_seen": 2159149056 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516767676767677, + "loss": 1.6672, + "theoretical_loss": 3.4087185466777763, + "tokens_seen": 2159214592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516565656565657, + "loss": 1.3043, + "theoretical_loss": 3.4087100498738536, + "tokens_seen": 2159280128 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035163636363636365, + "loss": 1.2514, + "theoretical_loss": 3.408701553400017, + "tokens_seen": 2159345664 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516161616161616, + "loss": 1.5195, + "theoretical_loss": 3.4086930572562446, + "tokens_seen": 2159411200 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035159595959595956, + "loss": 1.5695, + "theoretical_loss": 3.4086845614425134, + "tokens_seen": 2159476736 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515757575757576, + "loss": 1.4327, + "theoretical_loss": 3.4086760659588, + "tokens_seen": 2159542272 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515555555555556, + "loss": 1.633, + "theoretical_loss": 3.408667570805082, + "tokens_seen": 2159607808 + }, + { + "epoch": 0.3, + "objective/train/aligned_token_count": 0.4694525897502899, + "objective/train/avg_token_score": 0.0066945492289960384, + "objective/train/docs_used": 1223379, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5661883354187012, + "objective/train/original_loss": 4.237104415893555, + "objective/train/theoretical_loss": 3.408659075981337, + "objective/train/tokens_used": 518532576, + "objective/train/unlikelihood_loss": 0.050316303968429565, + "theoretical_loss": 3.408659075981337, + "tokens_seen": 2159673344 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035153535353535354, + "loss": 1.3222, + "theoretical_loss": 3.408659075981337, + "tokens_seen": 2159673344 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515151515151515, + "loss": 1.4225, + "theoretical_loss": 3.4086505814875414, + "tokens_seen": 2159738880 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035149494949494945, + "loss": 1.571, + "theoretical_loss": 3.408642087323673, + "tokens_seen": 2159804416 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514747474747475, + "loss": 1.6036, + "theoretical_loss": 3.4086335934897085, + "tokens_seen": 2159869952 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514545454545455, + "loss": 1.3161, + "theoretical_loss": 3.4086250999856254, + "tokens_seen": 2159935488 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035143434343434343, + "loss": 1.4086, + "theoretical_loss": 3.4086166068114006, + "tokens_seen": 2160001024 + } + ], + "max_steps": 25000, + "num_train_epochs": 9223372036854775807, + "total_flos": 2.5435211563008e+17, + "trial_name": null, + "trial_params": null +}