diff --git "a/checkpoint-5070/trainer_state.json" "b/checkpoint-5070/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-5070/trainer_state.json" @@ -0,0 +1,38550 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.2028, + "global_step": 5070, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 2e-06, + "loss": 2.3586, + "theoretical_loss": 3.484799908140819, + "tokens_seen": 1661665280 + }, + { + "epoch": 0.0, + "learning_rate": 4e-06, + "loss": 2.2339, + "theoretical_loss": 3.4847880270596066, + "tokens_seen": 1661730816 + }, + { + "epoch": 0.0, + "learning_rate": 6e-06, + "loss": 2.2016, + "theoretical_loss": 3.4847761465781497, + "tokens_seen": 1661796352 + }, + { + "epoch": 0.0, + "learning_rate": 8e-06, + "loss": 2.2196, + "theoretical_loss": 3.484764266696394, + "tokens_seen": 1661861888 + }, + { + "epoch": 0.0, + "learning_rate": 1e-05, + "loss": 2.1452, + "theoretical_loss": 3.4847523874142867, + "tokens_seen": 1661927424 + }, + { + "epoch": 0.0, + "learning_rate": 1.2e-05, + "loss": 2.1924, + "theoretical_loss": 3.4847405087317727, + "tokens_seen": 1661992960 + }, + { + "epoch": 0.0, + "learning_rate": 1.4e-05, + "loss": 2.3474, + "theoretical_loss": 3.4847286306487986, + "tokens_seen": 1662058496 + }, + { + "epoch": 0.0, + "learning_rate": 1.6e-05, + "loss": 2.3439, + "theoretical_loss": 3.48471675316531, + "tokens_seen": 1662124032 + }, + { + "epoch": 0.0, + "learning_rate": 1.8e-05, + "loss": 2.1693, + "theoretical_loss": 3.4847048762812536, + "tokens_seen": 1662189568 + }, + { + "epoch": 0.0, + "learning_rate": 2e-05, + "loss": 2.0245, + "theoretical_loss": 3.4846929999965757, + "tokens_seen": 1662255104 + }, + { + "epoch": 0.0, + "learning_rate": 2.2e-05, + "loss": 2.071, + "theoretical_loss": 3.484681124311222, + "tokens_seen": 1662320640 + }, + { + "epoch": 0.0, + "learning_rate": 2.4e-05, + "loss": 1.9012, + "theoretical_loss": 3.4846692492251385, + "tokens_seen": 1662386176 + }, + { + "epoch": 0.0, + "learning_rate": 2.6e-05, + "loss": 1.8901, + "theoretical_loss": 3.484657374738272, + "tokens_seen": 1662451712 + }, + { + "epoch": 0.0, + "learning_rate": 2.8e-05, + "loss": 1.9415, + "theoretical_loss": 3.4846455008505677, + "tokens_seen": 1662517248 + }, + { + "epoch": 0.0, + "learning_rate": 3e-05, + "loss": 1.9999, + "theoretical_loss": 3.484633627561972, + "tokens_seen": 1662582784 + }, + { + "epoch": 0.0, + "learning_rate": 3.2e-05, + "loss": 1.9898, + "theoretical_loss": 3.484621754872432, + "tokens_seen": 1662648320 + }, + { + "epoch": 0.0, + "learning_rate": 3.4000000000000007e-05, + "loss": 1.6089, + "theoretical_loss": 3.4846098827818928, + "tokens_seen": 1662713856 + }, + { + "epoch": 0.0, + "learning_rate": 3.6e-05, + "loss": 1.7149, + "theoretical_loss": 3.4845980112903003, + "tokens_seen": 1662779392 + }, + { + "epoch": 0.0, + "learning_rate": 3.8e-05, + "loss": 1.6944, + "theoretical_loss": 3.484586140397602, + "tokens_seen": 1662844928 + }, + { + "epoch": 0.0, + "learning_rate": 4e-05, + "loss": 1.4261, + "theoretical_loss": 3.484574270103743, + "tokens_seen": 1662910464 + }, + { + "epoch": 0.0, + "learning_rate": 4.2000000000000004e-05, + "loss": 1.7173, + "theoretical_loss": 3.484562400408671, + "tokens_seen": 1662976000 + }, + { + "epoch": 0.0, + "learning_rate": 4.4e-05, + "loss": 1.5526, + "theoretical_loss": 3.4845505313123297, + "tokens_seen": 1663041536 + }, + { + "epoch": 0.0, + "learning_rate": 4.6e-05, + "loss": 1.7101, + "theoretical_loss": 3.4845386628146677, + "tokens_seen": 1663107072 + }, + { + "epoch": 0.0, + "learning_rate": 4.8e-05, + "loss": 1.5717, + "theoretical_loss": 3.4845267949156296, + "tokens_seen": 1663172608 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.4392717480659485, + "objective/train/avg_token_score": 0.027796979993581772, + "objective/train/docs_used": 941634, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5729044675827026, + "objective/train/original_loss": 3.6452524662017822, + "objective/train/theoretical_loss": 3.4845149276151623, + "objective/train/tokens_used": 22097376, + "objective/train/unlikelihood_loss": 0.09146079421043396, + "theoretical_loss": 3.4845149276151623, + "tokens_seen": 1663238144 + }, + { + "epoch": 0.0, + "learning_rate": 5e-05, + "loss": 1.6885, + "theoretical_loss": 3.4845149276151623, + "tokens_seen": 1663238144 + }, + { + "epoch": 0.0, + "learning_rate": 5.2e-05, + "loss": 1.4219, + "theoretical_loss": 3.484503060913212, + "tokens_seen": 1663303680 + }, + { + "epoch": 0.0, + "learning_rate": 5.4e-05, + "loss": 1.5592, + "theoretical_loss": 3.484491194809725, + "tokens_seen": 1663369216 + }, + { + "epoch": 0.0, + "learning_rate": 5.6e-05, + "loss": 1.1071, + "theoretical_loss": 3.484479329304647, + "tokens_seen": 1663434752 + }, + { + "epoch": 0.0, + "learning_rate": 5.800000000000001e-05, + "loss": 1.4801, + "theoretical_loss": 3.484467464397925, + "tokens_seen": 1663500288 + }, + { + "epoch": 0.0, + "learning_rate": 6e-05, + "loss": 1.6332, + "theoretical_loss": 3.484455600089505, + "tokens_seen": 1663565824 + }, + { + "epoch": 0.0, + "learning_rate": 6.2e-05, + "loss": 1.4538, + "theoretical_loss": 3.4844437363793332, + "tokens_seen": 1663631360 + }, + { + "epoch": 0.0, + "learning_rate": 6.4e-05, + "loss": 1.5288, + "theoretical_loss": 3.4844318732673556, + "tokens_seen": 1663696896 + }, + { + "epoch": 0.0, + "learning_rate": 6.6e-05, + "loss": 1.4459, + "theoretical_loss": 3.484420010753519, + "tokens_seen": 1663762432 + }, + { + "epoch": 0.0, + "learning_rate": 6.800000000000001e-05, + "loss": 1.4075, + "theoretical_loss": 3.484408148837769, + "tokens_seen": 1663827968 + }, + { + "epoch": 0.0, + "learning_rate": 7.000000000000001e-05, + "loss": 1.2731, + "theoretical_loss": 3.4843962875200525, + "tokens_seen": 1663893504 + }, + { + "epoch": 0.0, + "learning_rate": 7.2e-05, + "loss": 1.2299, + "theoretical_loss": 3.484384426800316, + "tokens_seen": 1663959040 + }, + { + "epoch": 0.0, + "learning_rate": 7.4e-05, + "loss": 1.6132, + "theoretical_loss": 3.4843725666785046, + "tokens_seen": 1664024576 + }, + { + "epoch": 0.0, + "learning_rate": 7.6e-05, + "loss": 1.639, + "theoretical_loss": 3.484360707154566, + "tokens_seen": 1664090112 + }, + { + "epoch": 0.0, + "learning_rate": 7.8e-05, + "loss": 1.3998, + "theoretical_loss": 3.484348848228446, + "tokens_seen": 1664155648 + }, + { + "epoch": 0.0, + "learning_rate": 8e-05, + "loss": 1.6466, + "theoretical_loss": 3.4843369899000907, + "tokens_seen": 1664221184 + }, + { + "epoch": 0.0, + "learning_rate": 8.2e-05, + "loss": 1.4191, + "theoretical_loss": 3.484325132169446, + "tokens_seen": 1664286720 + }, + { + "epoch": 0.0, + "learning_rate": 8.400000000000001e-05, + "loss": 1.3433, + "theoretical_loss": 3.4843132750364596, + "tokens_seen": 1664352256 + }, + { + "epoch": 0.0, + "learning_rate": 8.599999999999999e-05, + "loss": 1.6113, + "theoretical_loss": 3.4843014185010768, + "tokens_seen": 1664417792 + }, + { + "epoch": 0.0, + "learning_rate": 8.8e-05, + "loss": 1.5214, + "theoretical_loss": 3.4842895625632444, + "tokens_seen": 1664483328 + }, + { + "epoch": 0.0, + "learning_rate": 8.999999999999999e-05, + "loss": 1.5353, + "theoretical_loss": 3.4842777072229083, + "tokens_seen": 1664548864 + }, + { + "epoch": 0.0, + "learning_rate": 9.2e-05, + "loss": 1.4214, + "theoretical_loss": 3.484265852480015, + "tokens_seen": 1664614400 + }, + { + "epoch": 0.0, + "learning_rate": 9.400000000000001e-05, + "loss": 1.4677, + "theoretical_loss": 3.4842539983345118, + "tokens_seen": 1664679936 + }, + { + "epoch": 0.0, + "learning_rate": 9.6e-05, + "loss": 1.4979, + "theoretical_loss": 3.484242144786344, + "tokens_seen": 1664745472 + }, + { + "epoch": 0.0, + "learning_rate": 9.800000000000001e-05, + "loss": 1.5655, + "theoretical_loss": 3.4842302918354577, + "tokens_seen": 1664811008 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.4052419364452362, + "objective/train/avg_token_score": 0.023708028718829155, + "objective/train/docs_used": 943199, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3439230918884277, + "objective/train/original_loss": 3.491624593734741, + "objective/train/theoretical_loss": 3.4842184394818, + "objective/train/tokens_used": 23735776, + "objective/train/unlikelihood_loss": 0.11052291095256805, + "theoretical_loss": 3.4842184394818, + "tokens_seen": 1664876544 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001, + "loss": 1.2824, + "theoretical_loss": 3.4842184394818, + "tokens_seen": 1664876544 + }, + { + "epoch": 0.0, + "learning_rate": 0.000102, + "loss": 1.3515, + "theoretical_loss": 3.4842065877253177, + "tokens_seen": 1664942080 + }, + { + "epoch": 0.0, + "learning_rate": 0.000104, + "loss": 1.327, + "theoretical_loss": 3.4841947365659562, + "tokens_seen": 1665007616 + }, + { + "epoch": 0.0, + "learning_rate": 0.000106, + "loss": 1.4169, + "theoretical_loss": 3.4841828860036625, + "tokens_seen": 1665073152 + }, + { + "epoch": 0.0, + "learning_rate": 0.000108, + "loss": 1.4514, + "theoretical_loss": 3.484171036038383, + "tokens_seen": 1665138688 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011, + "loss": 1.374, + "theoretical_loss": 3.4841591866700643, + "tokens_seen": 1665204224 + }, + { + "epoch": 0.0, + "learning_rate": 0.000112, + "loss": 1.5037, + "theoretical_loss": 3.4841473378986523, + "tokens_seen": 1665269760 + }, + { + "epoch": 0.0, + "learning_rate": 0.000114, + "loss": 1.5909, + "theoretical_loss": 3.4841354897240935, + "tokens_seen": 1665335296 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011600000000000001, + "loss": 1.3882, + "theoretical_loss": 3.4841236421463346, + "tokens_seen": 1665400832 + }, + { + "epoch": 0.0, + "learning_rate": 0.000118, + "loss": 1.4278, + "theoretical_loss": 3.4841117951653224, + "tokens_seen": 1665466368 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012, + "loss": 1.2305, + "theoretical_loss": 3.4840999487810027, + "tokens_seen": 1665531904 + }, + { + "epoch": 0.0, + "learning_rate": 0.000122, + "loss": 1.3588, + "theoretical_loss": 3.4840881029933226, + "tokens_seen": 1665597440 + }, + { + "epoch": 0.0, + "learning_rate": 0.000124, + "loss": 1.1805, + "theoretical_loss": 3.484076257802228, + "tokens_seen": 1665662976 + }, + { + "epoch": 0.0, + "learning_rate": 0.000126, + "loss": 1.4335, + "theoretical_loss": 3.4840644132076655, + "tokens_seen": 1665728512 + }, + { + "epoch": 0.0, + "learning_rate": 0.000128, + "loss": 1.5079, + "theoretical_loss": 3.484052569209582, + "tokens_seen": 1665794048 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013000000000000002, + "loss": 1.319, + "theoretical_loss": 3.4840407258079233, + "tokens_seen": 1665859584 + }, + { + "epoch": 0.0, + "learning_rate": 0.000132, + "loss": 1.4067, + "theoretical_loss": 3.4840288830026367, + "tokens_seen": 1665925120 + }, + { + "epoch": 0.0, + "learning_rate": 0.000134, + "loss": 1.2983, + "theoretical_loss": 3.484017040793668, + "tokens_seen": 1665990656 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013600000000000003, + "loss": 1.2986, + "theoretical_loss": 3.4840051991809644, + "tokens_seen": 1666056192 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013800000000000002, + "loss": 1.2998, + "theoretical_loss": 3.4839933581644718, + "tokens_seen": 1666121728 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014000000000000001, + "loss": 1.5612, + "theoretical_loss": 3.4839815177441373, + "tokens_seen": 1666187264 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014199999999999998, + "loss": 1.3289, + "theoretical_loss": 3.4839696779199065, + "tokens_seen": 1666252800 + }, + { + "epoch": 0.0, + "learning_rate": 0.000144, + "loss": 1.4071, + "theoretical_loss": 3.483957838691727, + "tokens_seen": 1666318336 + }, + { + "epoch": 0.0, + "learning_rate": 0.000146, + "loss": 1.1692, + "theoretical_loss": 3.483946000059545, + "tokens_seen": 1666383872 + }, + { + "epoch": 0.0, + "learning_rate": 0.000148, + "loss": 1.4058, + "theoretical_loss": 3.483934162023307, + "tokens_seen": 1666449408 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.3277125954627991, + "objective/train/avg_token_score": 0.01894918829202652, + "objective/train/docs_used": 943856, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0503660440444946, + "objective/train/original_loss": 3.4445393085479736, + "objective/train/theoretical_loss": 3.4839223245829594, + "objective/train/tokens_used": 25374176, + "objective/train/unlikelihood_loss": 0.0962013378739357, + "theoretical_loss": 3.4839223245829594, + "tokens_seen": 1666514944 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015, + "loss": 1.1905, + "theoretical_loss": 3.4839223245829594, + "tokens_seen": 1666514944 + }, + { + "epoch": 0.0, + "learning_rate": 0.000152, + "loss": 1.5246, + "theoretical_loss": 3.4839104877384486, + "tokens_seen": 1666580480 + }, + { + "epoch": 0.0, + "learning_rate": 0.000154, + "loss": 1.3905, + "theoretical_loss": 3.483898651489722, + "tokens_seen": 1666646016 + }, + { + "epoch": 0.0, + "learning_rate": 0.000156, + "loss": 1.4391, + "theoretical_loss": 3.4838868158367253, + "tokens_seen": 1666711552 + }, + { + "epoch": 0.0, + "learning_rate": 0.000158, + "loss": 1.2598, + "theoretical_loss": 3.4838749807794063, + "tokens_seen": 1666777088 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016, + "loss": 1.4413, + "theoretical_loss": 3.4838631463177103, + "tokens_seen": 1666842624 + }, + { + "epoch": 0.0, + "learning_rate": 0.000162, + "loss": 1.5131, + "theoretical_loss": 3.483851312451584, + "tokens_seen": 1666908160 + }, + { + "epoch": 0.0, + "learning_rate": 0.000164, + "loss": 1.333, + "theoretical_loss": 3.4838394791809746, + "tokens_seen": 1666973696 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016600000000000002, + "loss": 1.6326, + "theoretical_loss": 3.4838276465058287, + "tokens_seen": 1667039232 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016800000000000002, + "loss": 1.5681, + "theoretical_loss": 3.4838158144260927, + "tokens_seen": 1667104768 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017, + "loss": 1.1809, + "theoretical_loss": 3.483803982941714, + "tokens_seen": 1667170304 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017199999999999998, + "loss": 1.2542, + "theoretical_loss": 3.4837921520526374, + "tokens_seen": 1667235840 + }, + { + "epoch": 0.0, + "learning_rate": 0.000174, + "loss": 1.3605, + "theoretical_loss": 3.483780321758811, + "tokens_seen": 1667301376 + }, + { + "epoch": 0.0, + "learning_rate": 0.000176, + "loss": 1.6095, + "theoretical_loss": 3.4837684920601815, + "tokens_seen": 1667366912 + }, + { + "epoch": 0.0, + "learning_rate": 0.000178, + "loss": 1.4505, + "theoretical_loss": 3.483756662956695, + "tokens_seen": 1667432448 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017999999999999998, + "loss": 1.2638, + "theoretical_loss": 3.483744834448298, + "tokens_seen": 1667497984 + }, + { + "epoch": 0.0, + "learning_rate": 0.000182, + "loss": 1.4185, + "theoretical_loss": 3.483733006534938, + "tokens_seen": 1667563520 + }, + { + "epoch": 0.0, + "learning_rate": 0.000184, + "loss": 1.2676, + "theoretical_loss": 3.483721179216561, + "tokens_seen": 1667629056 + }, + { + "epoch": 0.0, + "learning_rate": 0.000186, + "loss": 1.7296, + "theoretical_loss": 3.4837093524931144, + "tokens_seen": 1667694592 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018800000000000002, + "loss": 1.6747, + "theoretical_loss": 3.4836975263645438, + "tokens_seen": 1667760128 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019, + "loss": 1.463, + "theoretical_loss": 3.483685700830797, + "tokens_seen": 1667825664 + }, + { + "epoch": 0.0, + "learning_rate": 0.000192, + "loss": 1.2903, + "theoretical_loss": 3.48367387589182, + "tokens_seen": 1667891200 + }, + { + "epoch": 0.0, + "learning_rate": 0.000194, + "loss": 1.3391, + "theoretical_loss": 3.48366205154756, + "tokens_seen": 1667956736 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019600000000000002, + "loss": 1.6197, + "theoretical_loss": 3.4836502277979626, + "tokens_seen": 1668022272 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019800000000000002, + "loss": 1.3482, + "theoretical_loss": 3.4836384046429765, + "tokens_seen": 1668087808 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.3819648027420044, + "objective/train/avg_token_score": 0.015858473256230354, + "objective/train/docs_used": 945126, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2602568864822388, + "objective/train/original_loss": 3.6047937870025635, + "objective/train/theoretical_loss": 3.4836265820825467, + "objective/train/tokens_used": 27012576, + "objective/train/unlikelihood_loss": 0.07870878279209137, + "theoretical_loss": 3.4836265820825467, + "tokens_seen": 1668153344 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002, + "loss": 1.3153, + "theoretical_loss": 3.4836265820825467, + "tokens_seen": 1668153344 + }, + { + "epoch": 0.0, + "learning_rate": 0.000202, + "loss": 1.313, + "theoretical_loss": 3.483614760116621, + "tokens_seen": 1668218880 + }, + { + "epoch": 0.0, + "learning_rate": 0.000204, + "loss": 1.4786, + "theoretical_loss": 3.4836029387451455, + "tokens_seen": 1668284416 + }, + { + "epoch": 0.0, + "learning_rate": 0.000206, + "loss": 1.4841, + "theoretical_loss": 3.483591117968067, + "tokens_seen": 1668349952 + }, + { + "epoch": 0.0, + "learning_rate": 0.000208, + "loss": 1.4115, + "theoretical_loss": 3.483579297785333, + "tokens_seen": 1668415488 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021, + "loss": 1.5871, + "theoretical_loss": 3.483567478196889, + "tokens_seen": 1668481024 + }, + { + "epoch": 0.0, + "learning_rate": 0.000212, + "loss": 1.5155, + "theoretical_loss": 3.483555659202683, + "tokens_seen": 1668546560 + }, + { + "epoch": 0.0, + "learning_rate": 0.000214, + "loss": 1.6009, + "theoretical_loss": 3.4835438408026613, + "tokens_seen": 1668612096 + }, + { + "epoch": 0.0, + "learning_rate": 0.000216, + "loss": 1.1647, + "theoretical_loss": 3.483532022996771, + "tokens_seen": 1668677632 + }, + { + "epoch": 0.0, + "learning_rate": 0.000218, + "loss": 1.3228, + "theoretical_loss": 3.483520205784958, + "tokens_seen": 1668743168 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022, + "loss": 1.4557, + "theoretical_loss": 3.48350838916717, + "tokens_seen": 1668808704 + }, + { + "epoch": 0.0, + "learning_rate": 0.000222, + "loss": 1.3735, + "theoretical_loss": 3.4834965731433534, + "tokens_seen": 1668874240 + }, + { + "epoch": 0.0, + "learning_rate": 0.000224, + "loss": 1.3874, + "theoretical_loss": 3.4834847577134553, + "tokens_seen": 1668939776 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022600000000000002, + "loss": 1.3996, + "theoretical_loss": 3.4834729428774223, + "tokens_seen": 1669005312 + }, + { + "epoch": 0.0, + "learning_rate": 0.000228, + "loss": 1.2483, + "theoretical_loss": 3.4834611286352013, + "tokens_seen": 1669070848 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023, + "loss": 1.5426, + "theoretical_loss": 3.4834493149867387, + "tokens_seen": 1669136384 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023200000000000003, + "loss": 1.1371, + "theoretical_loss": 3.483437501931982, + "tokens_seen": 1669201920 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023400000000000002, + "loss": 1.5659, + "theoretical_loss": 3.483425689470878, + "tokens_seen": 1669267456 + }, + { + "epoch": 0.0, + "learning_rate": 0.000236, + "loss": 1.5264, + "theoretical_loss": 3.4834138776033736, + "tokens_seen": 1669332992 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023799999999999998, + "loss": 1.3228, + "theoretical_loss": 3.483402066329415, + "tokens_seen": 1669398528 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024, + "loss": 1.4733, + "theoretical_loss": 3.48339025564895, + "tokens_seen": 1669464064 + }, + { + "epoch": 0.0, + "learning_rate": 0.000242, + "loss": 1.329, + "theoretical_loss": 3.4833784455619248, + "tokens_seen": 1669529600 + }, + { + "epoch": 0.0, + "learning_rate": 0.000244, + "loss": 1.4319, + "theoretical_loss": 3.4833666360682867, + "tokens_seen": 1669595136 + }, + { + "epoch": 0.0, + "learning_rate": 0.000246, + "loss": 1.4501, + "theoretical_loss": 3.4833548271679824, + "tokens_seen": 1669660672 + }, + { + "epoch": 0.0, + "learning_rate": 0.000248, + "loss": 1.3319, + "theoretical_loss": 3.4833430188609587, + "tokens_seen": 1669726208 + }, + { + "epoch": 0.0, + "objective/train/aligned_token_count": 0.45992180705070496, + "objective/train/avg_token_score": 0.009118808433413506, + "objective/train/docs_used": 945699, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6401410102844238, + "objective/train/original_loss": 4.021496772766113, + "objective/train/theoretical_loss": 3.4833312111471626, + "objective/train/tokens_used": 28650976, + "objective/train/unlikelihood_loss": 0.06683876365423203, + "theoretical_loss": 3.4833312111471626, + "tokens_seen": 1669791744 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025, + "loss": 1.6621, + "theoretical_loss": 3.4833312111471626, + "tokens_seen": 1669791744 + }, + { + "epoch": 0.01, + "learning_rate": 0.000252, + "loss": 1.361, + "theoretical_loss": 3.483319404026541, + "tokens_seen": 1669857280 + }, + { + "epoch": 0.01, + "learning_rate": 0.000254, + "loss": 1.1887, + "theoretical_loss": 3.4833075974990404, + "tokens_seen": 1669922816 + }, + { + "epoch": 0.01, + "learning_rate": 0.000256, + "loss": 1.4238, + "theoretical_loss": 3.4832957915646086, + "tokens_seen": 1669988352 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025800000000000004, + "loss": 1.4582, + "theoretical_loss": 3.4832839862231926, + "tokens_seen": 1670053888 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026000000000000003, + "loss": 1.509, + "theoretical_loss": 3.4832721814747383, + "tokens_seen": 1670119424 + }, + { + "epoch": 0.01, + "learning_rate": 0.000262, + "loss": 1.6246, + "theoretical_loss": 3.4832603773191932, + "tokens_seen": 1670184960 + }, + { + "epoch": 0.01, + "learning_rate": 0.000264, + "loss": 1.3161, + "theoretical_loss": 3.4832485737565047, + "tokens_seen": 1670250496 + }, + { + "epoch": 0.01, + "learning_rate": 0.000266, + "loss": 1.6053, + "theoretical_loss": 3.483236770786619, + "tokens_seen": 1670316032 + }, + { + "epoch": 0.01, + "learning_rate": 0.000268, + "loss": 1.2865, + "theoretical_loss": 3.4832249684094836, + "tokens_seen": 1670381568 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027, + "loss": 1.3504, + "theoretical_loss": 3.4832131666250454, + "tokens_seen": 1670447104 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027200000000000005, + "loss": 1.4324, + "theoretical_loss": 3.483201365433251, + "tokens_seen": 1670512640 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027400000000000005, + "loss": 1.478, + "theoretical_loss": 3.4831895648340474, + "tokens_seen": 1670578176 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027600000000000004, + "loss": 1.3201, + "theoretical_loss": 3.4831777648273823, + "tokens_seen": 1670643712 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027800000000000004, + "loss": 1.3004, + "theoretical_loss": 3.4831659654132023, + "tokens_seen": 1670709248 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028000000000000003, + "loss": 1.2886, + "theoretical_loss": 3.4831541665914543, + "tokens_seen": 1670774784 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028199999999999997, + "loss": 1.4002, + "theoretical_loss": 3.4831423683620852, + "tokens_seen": 1670840320 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028399999999999996, + "loss": 1.0925, + "theoretical_loss": 3.4831305707250424, + "tokens_seen": 1670905856 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028599999999999996, + "loss": 1.594, + "theoretical_loss": 3.483118773680273, + "tokens_seen": 1670971392 + }, + { + "epoch": 0.01, + "learning_rate": 0.000288, + "loss": 1.5114, + "theoretical_loss": 3.4831069772277234, + "tokens_seen": 1671036928 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029, + "loss": 1.4329, + "theoretical_loss": 3.483095181367341, + "tokens_seen": 1671102464 + }, + { + "epoch": 0.01, + "learning_rate": 0.000292, + "loss": 1.5268, + "theoretical_loss": 3.4830833860990733, + "tokens_seen": 1671168000 + }, + { + "epoch": 0.01, + "learning_rate": 0.000294, + "loss": 1.3822, + "theoretical_loss": 3.4830715914228665, + "tokens_seen": 1671233536 + }, + { + "epoch": 0.01, + "learning_rate": 0.000296, + "loss": 1.5873, + "theoretical_loss": 3.4830597973386683, + "tokens_seen": 1671299072 + }, + { + "epoch": 0.01, + "learning_rate": 0.000298, + "loss": 1.5036, + "theoretical_loss": 3.4830480038464255, + "tokens_seen": 1671364608 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.3960166275501251, + "objective/train/avg_token_score": 0.020477375015616417, + "objective/train/docs_used": 946982, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2867282629013062, + "objective/train/original_loss": 4.248522758483887, + "objective/train/theoretical_loss": 3.4830362109460857, + "objective/train/tokens_used": 30289376, + "objective/train/unlikelihood_loss": 0.06127317622303963, + "theoretical_loss": 3.4830362109460857, + "tokens_seen": 1671430144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003, + "loss": 1.2874, + "theoretical_loss": 3.4830362109460857, + "tokens_seen": 1671430144 + }, + { + "epoch": 0.01, + "learning_rate": 0.000302, + "loss": 1.5552, + "theoretical_loss": 3.483024418637595, + "tokens_seen": 1671495680 + }, + { + "epoch": 0.01, + "learning_rate": 0.000304, + "loss": 1.4532, + "theoretical_loss": 3.4830126269209014, + "tokens_seen": 1671561216 + }, + { + "epoch": 0.01, + "learning_rate": 0.000306, + "loss": 1.3209, + "theoretical_loss": 3.4830008357959517, + "tokens_seen": 1671626752 + }, + { + "epoch": 0.01, + "learning_rate": 0.000308, + "loss": 1.5412, + "theoretical_loss": 3.4829890452626926, + "tokens_seen": 1671692288 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031, + "loss": 1.0823, + "theoretical_loss": 3.482977255321072, + "tokens_seen": 1671757824 + }, + { + "epoch": 0.01, + "learning_rate": 0.000312, + "loss": 1.4433, + "theoretical_loss": 3.482965465971036, + "tokens_seen": 1671823360 + }, + { + "epoch": 0.01, + "learning_rate": 0.000314, + "loss": 1.3343, + "theoretical_loss": 3.4829536772125325, + "tokens_seen": 1671888896 + }, + { + "epoch": 0.01, + "learning_rate": 0.000316, + "loss": 1.4329, + "theoretical_loss": 3.482941889045509, + "tokens_seen": 1671954432 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031800000000000003, + "loss": 1.2974, + "theoretical_loss": 3.4829301014699117, + "tokens_seen": 1672019968 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032, + "loss": 1.5543, + "theoretical_loss": 3.482918314485688, + "tokens_seen": 1672085504 + }, + { + "epoch": 0.01, + "learning_rate": 0.000322, + "loss": 1.6948, + "theoretical_loss": 3.4829065280927853, + "tokens_seen": 1672151040 + }, + { + "epoch": 0.01, + "learning_rate": 0.000324, + "loss": 1.358, + "theoretical_loss": 3.482894742291151, + "tokens_seen": 1672216576 + }, + { + "epoch": 0.01, + "learning_rate": 0.000326, + "loss": 1.5256, + "theoretical_loss": 3.4828829570807316, + "tokens_seen": 1672282112 + }, + { + "epoch": 0.01, + "learning_rate": 0.000328, + "loss": 1.2557, + "theoretical_loss": 3.4828711724614747, + "tokens_seen": 1672347648 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033, + "loss": 1.4626, + "theoretical_loss": 3.4828593884333277, + "tokens_seen": 1672413184 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033200000000000005, + "loss": 1.4653, + "theoretical_loss": 3.4828476049962367, + "tokens_seen": 1672478720 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033400000000000004, + "loss": 1.4492, + "theoretical_loss": 3.4828358221501503, + "tokens_seen": 1672544256 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033600000000000004, + "loss": 1.3397, + "theoretical_loss": 3.4828240398950143, + "tokens_seen": 1672609792 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033800000000000003, + "loss": 1.2303, + "theoretical_loss": 3.4828122582307777, + "tokens_seen": 1672675328 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034, + "loss": 1.3492, + "theoretical_loss": 3.482800477157386, + "tokens_seen": 1672740864 + }, + { + "epoch": 0.01, + "learning_rate": 0.000342, + "loss": 1.4296, + "theoretical_loss": 3.4827886966747874, + "tokens_seen": 1672806400 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034399999999999996, + "loss": 1.6639, + "theoretical_loss": 3.482776916782929, + "tokens_seen": 1672871936 + }, + { + "epoch": 0.01, + "learning_rate": 0.000346, + "loss": 1.3111, + "theoretical_loss": 3.482765137481757, + "tokens_seen": 1672937472 + }, + { + "epoch": 0.01, + "learning_rate": 0.000348, + "loss": 1.4424, + "theoretical_loss": 3.4827533587712205, + "tokens_seen": 1673003008 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.42277616262435913, + "objective/train/avg_token_score": 0.034132879227399826, + "objective/train/docs_used": 947668, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4029995203018188, + "objective/train/original_loss": 4.207383155822754, + "objective/train/theoretical_loss": 3.4827415806512656, + "objective/train/tokens_used": 31927776, + "objective/train/unlikelihood_loss": 0.05666952580213547, + "theoretical_loss": 3.4827415806512656, + "tokens_seen": 1673068544 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035, + "loss": 1.7249, + "theoretical_loss": 3.4827415806512656, + "tokens_seen": 1673068544 + }, + { + "epoch": 0.01, + "learning_rate": 0.000352, + "loss": 1.4411, + "theoretical_loss": 3.4827298031218397, + "tokens_seen": 1673134080 + }, + { + "epoch": 0.01, + "learning_rate": 0.000354, + "loss": 1.4599, + "theoretical_loss": 3.48271802618289, + "tokens_seen": 1673199616 + }, + { + "epoch": 0.01, + "learning_rate": 0.000356, + "loss": 1.4334, + "theoretical_loss": 3.4827062498343637, + "tokens_seen": 1673265152 + }, + { + "epoch": 0.01, + "learning_rate": 0.000358, + "loss": 1.4262, + "theoretical_loss": 3.4826944740762085, + "tokens_seen": 1673330688 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035999999999999997, + "loss": 1.5422, + "theoretical_loss": 3.4826826989083712, + "tokens_seen": 1673396224 + }, + { + "epoch": 0.01, + "learning_rate": 0.000362, + "loss": 1.3631, + "theoretical_loss": 3.4826709243307996, + "tokens_seen": 1673461760 + }, + { + "epoch": 0.01, + "learning_rate": 0.000364, + "loss": 1.5299, + "theoretical_loss": 3.4826591503434408, + "tokens_seen": 1673527296 + }, + { + "epoch": 0.01, + "learning_rate": 0.000366, + "loss": 1.4358, + "theoretical_loss": 3.4826473769462414, + "tokens_seen": 1673592832 + }, + { + "epoch": 0.01, + "learning_rate": 0.000368, + "loss": 1.3031, + "theoretical_loss": 3.4826356041391495, + "tokens_seen": 1673658368 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037, + "loss": 1.372, + "theoretical_loss": 3.482623831922113, + "tokens_seen": 1673723904 + }, + { + "epoch": 0.01, + "learning_rate": 0.000372, + "loss": 1.4469, + "theoretical_loss": 3.4826120602950774, + "tokens_seen": 1673789440 + }, + { + "epoch": 0.01, + "learning_rate": 0.000374, + "loss": 1.3485, + "theoretical_loss": 3.4826002892579915, + "tokens_seen": 1673854976 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037600000000000003, + "loss": 1.1995, + "theoretical_loss": 3.482588518810802, + "tokens_seen": 1673920512 + }, + { + "epoch": 0.01, + "learning_rate": 0.000378, + "loss": 1.4059, + "theoretical_loss": 3.4825767489534574, + "tokens_seen": 1673986048 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038, + "loss": 1.4645, + "theoretical_loss": 3.482564979685903, + "tokens_seen": 1674051584 + }, + { + "epoch": 0.01, + "learning_rate": 0.000382, + "loss": 1.6177, + "theoretical_loss": 3.482553211008088, + "tokens_seen": 1674117120 + }, + { + "epoch": 0.01, + "learning_rate": 0.000384, + "loss": 1.5993, + "theoretical_loss": 3.482541442919959, + "tokens_seen": 1674182656 + }, + { + "epoch": 0.01, + "learning_rate": 0.000386, + "loss": 1.5792, + "theoretical_loss": 3.482529675421463, + "tokens_seen": 1674248192 + }, + { + "epoch": 0.01, + "learning_rate": 0.000388, + "loss": 1.2894, + "theoretical_loss": 3.4825179085125484, + "tokens_seen": 1674313728 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039000000000000005, + "loss": 1.3151, + "theoretical_loss": 3.4825061421931616, + "tokens_seen": 1674379264 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039200000000000004, + "loss": 1.484, + "theoretical_loss": 3.4824943764632503, + "tokens_seen": 1674444800 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039400000000000004, + "loss": 1.4304, + "theoretical_loss": 3.4824826113227623, + "tokens_seen": 1674510336 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039600000000000003, + "loss": 1.3797, + "theoretical_loss": 3.4824708467716445, + "tokens_seen": 1674575872 + }, + { + "epoch": 0.01, + "learning_rate": 0.000398, + "loss": 1.1721, + "theoretical_loss": 3.4824590828098447, + "tokens_seen": 1674641408 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.47403469681739807, + "objective/train/avg_token_score": 0.020986413583159447, + "objective/train/docs_used": 948408, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4957624673843384, + "objective/train/original_loss": 4.087368488311768, + "objective/train/theoretical_loss": 3.48244731943731, + "objective/train/tokens_used": 33566176, + "objective/train/unlikelihood_loss": 0.058626897633075714, + "theoretical_loss": 3.48244731943731, + "tokens_seen": 1674706944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004, + "loss": 1.6842, + "theoretical_loss": 3.48244731943731, + "tokens_seen": 1674706944 + }, + { + "epoch": 0.01, + "learning_rate": 0.000402, + "loss": 1.3746, + "theoretical_loss": 3.4824355566539875, + "tokens_seen": 1674772480 + }, + { + "epoch": 0.01, + "learning_rate": 0.000404, + "loss": 1.4025, + "theoretical_loss": 3.4824237944598257, + "tokens_seen": 1674838016 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040600000000000006, + "loss": 1.4322, + "theoretical_loss": 3.482412032854771, + "tokens_seen": 1674903552 + }, + { + "epoch": 0.01, + "learning_rate": 0.000408, + "loss": 1.5092, + "theoretical_loss": 3.482400271838771, + "tokens_seen": 1674969088 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041, + "loss": 1.3048, + "theoretical_loss": 3.482388511411774, + "tokens_seen": 1675034624 + }, + { + "epoch": 0.01, + "learning_rate": 0.000412, + "loss": 1.1327, + "theoretical_loss": 3.4823767515737267, + "tokens_seen": 1675100160 + }, + { + "epoch": 0.01, + "learning_rate": 0.000414, + "loss": 1.3768, + "theoretical_loss": 3.4823649923245767, + "tokens_seen": 1675165696 + }, + { + "epoch": 0.01, + "learning_rate": 0.000416, + "loss": 1.3944, + "theoretical_loss": 3.4823532336642717, + "tokens_seen": 1675231232 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041799999999999997, + "loss": 1.5612, + "theoretical_loss": 3.482341475592759, + "tokens_seen": 1675296768 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042, + "loss": 1.6033, + "theoretical_loss": 3.4823297181099857, + "tokens_seen": 1675362304 + }, + { + "epoch": 0.01, + "learning_rate": 0.000422, + "loss": 1.2907, + "theoretical_loss": 3.4823179612159, + "tokens_seen": 1675427840 + }, + { + "epoch": 0.01, + "learning_rate": 0.000424, + "loss": 1.4496, + "theoretical_loss": 3.4823062049104485, + "tokens_seen": 1675493376 + }, + { + "epoch": 0.01, + "learning_rate": 0.000426, + "loss": 1.3219, + "theoretical_loss": 3.4822944491935797, + "tokens_seen": 1675558912 + }, + { + "epoch": 0.01, + "learning_rate": 0.000428, + "loss": 1.6165, + "theoretical_loss": 3.4822826940652405, + "tokens_seen": 1675624448 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043, + "loss": 1.46, + "theoretical_loss": 3.482270939525379, + "tokens_seen": 1675689984 + }, + { + "epoch": 0.01, + "learning_rate": 0.000432, + "loss": 1.3409, + "theoretical_loss": 3.482259185573942, + "tokens_seen": 1675755520 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043400000000000003, + "loss": 1.5918, + "theoretical_loss": 3.4822474322108774, + "tokens_seen": 1675821056 + }, + { + "epoch": 0.01, + "learning_rate": 0.000436, + "loss": 1.3411, + "theoretical_loss": 3.4822356794361324, + "tokens_seen": 1675886592 + }, + { + "epoch": 0.01, + "learning_rate": 0.000438, + "loss": 1.4671, + "theoretical_loss": 3.482223927249655, + "tokens_seen": 1675952128 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044, + "loss": 1.407, + "theoretical_loss": 3.482212175651393, + "tokens_seen": 1676017664 + }, + { + "epoch": 0.01, + "learning_rate": 0.000442, + "loss": 1.4158, + "theoretical_loss": 3.4822004246412934, + "tokens_seen": 1676083200 + }, + { + "epoch": 0.01, + "learning_rate": 0.000444, + "loss": 1.2442, + "theoretical_loss": 3.4821886742193042, + "tokens_seen": 1676148736 + }, + { + "epoch": 0.01, + "learning_rate": 0.000446, + "loss": 1.6039, + "theoretical_loss": 3.4821769243853717, + "tokens_seen": 1676214272 + }, + { + "epoch": 0.01, + "learning_rate": 0.000448, + "loss": 1.3757, + "theoretical_loss": 3.4821651751394453, + "tokens_seen": 1676279808 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.43438416719436646, + "objective/train/avg_token_score": 0.02250559814274311, + "objective/train/docs_used": 949481, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3553886413574219, + "objective/train/original_loss": 4.167667388916016, + "objective/train/theoretical_loss": 3.4821534264814717, + "objective/train/tokens_used": 35204576, + "objective/train/unlikelihood_loss": 0.05399244651198387, + "theoretical_loss": 3.4821534264814717, + "tokens_seen": 1676345344 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045000000000000004, + "loss": 1.5774, + "theoretical_loss": 3.4821534264814717, + "tokens_seen": 1676345344 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045200000000000004, + "loss": 1.1835, + "theoretical_loss": 3.4821416784113985, + "tokens_seen": 1676410880 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045400000000000003, + "loss": 1.4729, + "theoretical_loss": 3.4821299309291733, + "tokens_seen": 1676476416 + }, + { + "epoch": 0.01, + "learning_rate": 0.000456, + "loss": 1.4368, + "theoretical_loss": 3.4821181840347437, + "tokens_seen": 1676541952 + }, + { + "epoch": 0.01, + "learning_rate": 0.000458, + "loss": 1.5042, + "theoretical_loss": 3.4821064377280577, + "tokens_seen": 1676607488 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046, + "loss": 1.4255, + "theoretical_loss": 3.4820946920090625, + "tokens_seen": 1676673024 + }, + { + "epoch": 0.01, + "learning_rate": 0.000462, + "loss": 1.3812, + "theoretical_loss": 3.4820829468777057, + "tokens_seen": 1676738560 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046400000000000006, + "loss": 1.384, + "theoretical_loss": 3.482071202333935, + "tokens_seen": 1676804096 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046600000000000005, + "loss": 1.3564, + "theoretical_loss": 3.482059458377698, + "tokens_seen": 1676869632 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046800000000000005, + "loss": 1.4955, + "theoretical_loss": 3.4820477150089433, + "tokens_seen": 1676935168 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047, + "loss": 1.3263, + "theoretical_loss": 3.4820359722276164, + "tokens_seen": 1677000704 + }, + { + "epoch": 0.01, + "learning_rate": 0.000472, + "loss": 1.6779, + "theoretical_loss": 3.4820242300336677, + "tokens_seen": 1677066240 + }, + { + "epoch": 0.01, + "learning_rate": 0.000474, + "loss": 1.4524, + "theoretical_loss": 3.4820124884270425, + "tokens_seen": 1677131776 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047599999999999997, + "loss": 1.5956, + "theoretical_loss": 3.4820007474076893, + "tokens_seen": 1677197312 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047799999999999996, + "loss": 1.5913, + "theoretical_loss": 3.481989006975556, + "tokens_seen": 1677262848 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048, + "loss": 1.4133, + "theoretical_loss": 3.4819772671305906, + "tokens_seen": 1677328384 + }, + { + "epoch": 0.01, + "learning_rate": 0.000482, + "loss": 1.2181, + "theoretical_loss": 3.48196552787274, + "tokens_seen": 1677393920 + }, + { + "epoch": 0.01, + "learning_rate": 0.000484, + "loss": 1.337, + "theoretical_loss": 3.4819537892019525, + "tokens_seen": 1677459456 + }, + { + "epoch": 0.01, + "learning_rate": 0.000486, + "loss": 1.2865, + "theoretical_loss": 3.4819420511181756, + "tokens_seen": 1677524992 + }, + { + "epoch": 0.01, + "learning_rate": 0.000488, + "loss": 1.6925, + "theoretical_loss": 3.481930313621357, + "tokens_seen": 1677590528 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049, + "loss": 1.2952, + "theoretical_loss": 3.4819185767114442, + "tokens_seen": 1677656064 + }, + { + "epoch": 0.01, + "learning_rate": 0.000492, + "loss": 1.3687, + "theoretical_loss": 3.4819068403883855, + "tokens_seen": 1677721600 + }, + { + "epoch": 0.01, + "learning_rate": 0.000494, + "loss": 1.4566, + "theoretical_loss": 3.4818951046521276, + "tokens_seen": 1677787136 + }, + { + "epoch": 0.01, + "learning_rate": 0.000496, + "loss": 1.3783, + "theoretical_loss": 3.4818833695026195, + "tokens_seen": 1677852672 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498, + "loss": 1.4967, + "theoretical_loss": 3.4818716349398082, + "tokens_seen": 1677918208 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.42913001775741577, + "objective/train/avg_token_score": 0.028014685958623886, + "objective/train/docs_used": 950694, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2823351621627808, + "objective/train/original_loss": 4.018346309661865, + "objective/train/theoretical_loss": 3.481859900963642, + "objective/train/tokens_used": 36842976, + "objective/train/unlikelihood_loss": 0.06534267216920853, + "theoretical_loss": 3.481859900963642, + "tokens_seen": 1677983744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005, + "loss": 1.4589, + "theoretical_loss": 3.481859900963642, + "tokens_seen": 1677983744 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499979797979798, + "loss": 1.7066, + "theoretical_loss": 3.481848167574068, + "tokens_seen": 1678049280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999595959595959, + "loss": 1.4743, + "theoretical_loss": 3.481836434771034, + "tokens_seen": 1678114816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999393939393939, + "loss": 1.4814, + "theoretical_loss": 3.4818247025544884, + "tokens_seen": 1678180352 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499919191919192, + "loss": 1.5105, + "theoretical_loss": 3.481812970924379, + "tokens_seen": 1678245888 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049989898989899, + "loss": 1.3163, + "theoretical_loss": 3.481801239880652, + "tokens_seen": 1678311424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998787878787879, + "loss": 1.5049, + "theoretical_loss": 3.4817895094232574, + "tokens_seen": 1678376960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998585858585859, + "loss": 1.7115, + "theoretical_loss": 3.481777779552141, + "tokens_seen": 1678442496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998383838383839, + "loss": 1.3624, + "theoretical_loss": 3.481766050267253, + "tokens_seen": 1678508032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998181818181818, + "loss": 1.4109, + "theoretical_loss": 3.481754321568539, + "tokens_seen": 1678573568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997979797979798, + "loss": 1.6151, + "theoretical_loss": 3.4817425934559476, + "tokens_seen": 1678639104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997777777777778, + "loss": 1.4839, + "theoretical_loss": 3.4817308659294266, + "tokens_seen": 1678704640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997575757575757, + "loss": 1.3913, + "theoretical_loss": 3.4817191389889244, + "tokens_seen": 1678770176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997373737373737, + "loss": 1.4744, + "theoretical_loss": 3.481707412634388, + "tokens_seen": 1678835712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997171717171717, + "loss": 1.2652, + "theoretical_loss": 3.481695686865766, + "tokens_seen": 1678901248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996969696969697, + "loss": 1.5878, + "theoretical_loss": 3.481683961683005, + "tokens_seen": 1678966784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996767676767676, + "loss": 1.2625, + "theoretical_loss": 3.4816722370860544, + "tokens_seen": 1679032320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996565656565657, + "loss": 1.3929, + "theoretical_loss": 3.4816605130748615, + "tokens_seen": 1679097856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996363636363637, + "loss": 1.4169, + "theoretical_loss": 3.4816487896493733, + "tokens_seen": 1679163392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996161616161617, + "loss": 1.4403, + "theoretical_loss": 3.4816370668095393, + "tokens_seen": 1679228928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995959595959596, + "loss": 1.5393, + "theoretical_loss": 3.4816253445553063, + "tokens_seen": 1679294464 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995757575757576, + "loss": 1.6212, + "theoretical_loss": 3.4816136228866217, + "tokens_seen": 1679360000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995555555555555, + "loss": 1.4312, + "theoretical_loss": 3.481601901803435, + "tokens_seen": 1679425536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995353535353536, + "loss": 1.4936, + "theoretical_loss": 3.481590181305693, + "tokens_seen": 1679491072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995151515151515, + "loss": 1.29, + "theoretical_loss": 3.4815784613933434, + "tokens_seen": 1679556608 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.3876466155052185, + "objective/train/avg_token_score": 0.019771317020058632, + "objective/train/docs_used": 951339, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2079968452453613, + "objective/train/original_loss": 3.654224395751953, + "objective/train/theoretical_loss": 3.481566742066335, + "objective/train/tokens_used": 38481376, + "objective/train/unlikelihood_loss": 0.10271399468183517, + "theoretical_loss": 3.481566742066335, + "tokens_seen": 1679622144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994949494949495, + "loss": 1.1997, + "theoretical_loss": 3.481566742066335, + "tokens_seen": 1679622144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994747474747474, + "loss": 1.3423, + "theoretical_loss": 3.4815550233246153, + "tokens_seen": 1679687680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994545454545454, + "loss": 1.416, + "theoretical_loss": 3.481543305168132, + "tokens_seen": 1679753216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994343434343435, + "loss": 1.4918, + "theoretical_loss": 3.4815315875968333, + "tokens_seen": 1679818752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994141414141415, + "loss": 1.6278, + "theoretical_loss": 3.4815198706106676, + "tokens_seen": 1679884288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993939393939394, + "loss": 1.4463, + "theoretical_loss": 3.481508154209582, + "tokens_seen": 1679949824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993737373737374, + "loss": 1.4316, + "theoretical_loss": 3.4814964383935245, + "tokens_seen": 1680015360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993535353535354, + "loss": 1.5408, + "theoretical_loss": 3.4814847231624437, + "tokens_seen": 1680080896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993333333333334, + "loss": 1.4201, + "theoretical_loss": 3.4814730085162875, + "tokens_seen": 1680146432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993131313131313, + "loss": 1.3325, + "theoretical_loss": 3.4814612944550034, + "tokens_seen": 1680211968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992929292929293, + "loss": 1.4038, + "theoretical_loss": 3.4814495809785395, + "tokens_seen": 1680277504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992727272727272, + "loss": 1.4769, + "theoretical_loss": 3.4814378680868447, + "tokens_seen": 1680343040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992525252525252, + "loss": 1.5646, + "theoretical_loss": 3.4814261557798654, + "tokens_seen": 1680408576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992323232323232, + "loss": 1.3709, + "theoretical_loss": 3.4814144440575507, + "tokens_seen": 1680474112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992121212121212, + "loss": 1.3969, + "theoretical_loss": 3.4814027329198485, + "tokens_seen": 1680539648 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991919191919192, + "loss": 1.4675, + "theoretical_loss": 3.481391022366707, + "tokens_seen": 1680605184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991717171717172, + "loss": 1.5211, + "theoretical_loss": 3.481379312398073, + "tokens_seen": 1680670720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991515151515152, + "loss": 1.424, + "theoretical_loss": 3.4813676030138963, + "tokens_seen": 1680736256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991313131313132, + "loss": 1.285, + "theoretical_loss": 3.481355894214124, + "tokens_seen": 1680801792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991111111111111, + "loss": 1.5255, + "theoretical_loss": 3.481344185998704, + "tokens_seen": 1680867328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990909090909091, + "loss": 1.3467, + "theoretical_loss": 3.4813324783675847, + "tokens_seen": 1680932864 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990707070707071, + "loss": 1.414, + "theoretical_loss": 3.481320771320714, + "tokens_seen": 1680998400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990505050505051, + "loss": 1.4323, + "theoretical_loss": 3.4813090648580403, + "tokens_seen": 1681063936 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499030303030303, + "loss": 1.2216, + "theoretical_loss": 3.4812973589795106, + "tokens_seen": 1681129472 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499010101010101, + "loss": 1.6606, + "theoretical_loss": 3.4812856536850747, + "tokens_seen": 1681195008 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.57746821641922, + "objective/train/avg_token_score": 0.003087435383349657, + "objective/train/docs_used": 952727, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.965943694114685, + "objective/train/original_loss": 3.664557933807373, + "objective/train/theoretical_loss": 3.4812739489746796, + "objective/train/tokens_used": 40119776, + "objective/train/unlikelihood_loss": 0.07379762828350067, + "theoretical_loss": 3.4812739489746796, + "tokens_seen": 1681260544 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498989898989899, + "loss": 1.3937, + "theoretical_loss": 3.4812739489746796, + "tokens_seen": 1681260544 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498969696969697, + "loss": 1.4396, + "theoretical_loss": 3.4812622448482733, + "tokens_seen": 1681326080 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498949494949495, + "loss": 1.5074, + "theoretical_loss": 3.481250541305804, + "tokens_seen": 1681391616 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498929292929293, + "loss": 1.3075, + "theoretical_loss": 3.4812388383472204, + "tokens_seen": 1681457152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989090909090909, + "loss": 1.4787, + "theoretical_loss": 3.4812271359724702, + "tokens_seen": 1681522688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988888888888889, + "loss": 1.4177, + "theoretical_loss": 3.481215434181501, + "tokens_seen": 1681588224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988686868686869, + "loss": 1.3858, + "theoretical_loss": 3.481203732974262, + "tokens_seen": 1681653760 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988484848484849, + "loss": 1.5262, + "theoretical_loss": 3.481192032350701, + "tokens_seen": 1681719296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988282828282828, + "loss": 1.5809, + "theoretical_loss": 3.481180332310765, + "tokens_seen": 1681784832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988080808080808, + "loss": 1.505, + "theoretical_loss": 3.481168632854404, + "tokens_seen": 1681850368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987878787878788, + "loss": 1.3779, + "theoretical_loss": 3.4811569339815644, + "tokens_seen": 1681915904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987676767676767, + "loss": 1.3716, + "theoretical_loss": 3.4811452356921953, + "tokens_seen": 1681981440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987474747474747, + "loss": 1.572, + "theoretical_loss": 3.4811335379862456, + "tokens_seen": 1682046976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987272727272728, + "loss": 1.4112, + "theoretical_loss": 3.4811218408636617, + "tokens_seen": 1682112512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987070707070708, + "loss": 1.4665, + "theoretical_loss": 3.481110144324393, + "tokens_seen": 1682178048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986868686868687, + "loss": 1.4609, + "theoretical_loss": 3.481098448368387, + "tokens_seen": 1682243584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986666666666667, + "loss": 1.6894, + "theoretical_loss": 3.4810867529955933, + "tokens_seen": 1682309120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986464646464647, + "loss": 1.3648, + "theoretical_loss": 3.481075058205958, + "tokens_seen": 1682374656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986262626262627, + "loss": 1.5816, + "theoretical_loss": 3.4810633639994313, + "tokens_seen": 1682440192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986060606060606, + "loss": 1.6544, + "theoretical_loss": 3.48105167037596, + "tokens_seen": 1682505728 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985858585858586, + "loss": 1.4945, + "theoretical_loss": 3.4810399773354925, + "tokens_seen": 1682571264 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985656565656565, + "loss": 1.3454, + "theoretical_loss": 3.4810282848779783, + "tokens_seen": 1682636800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985454545454545, + "loss": 1.2652, + "theoretical_loss": 3.481016593003364, + "tokens_seen": 1682702336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985252525252525, + "loss": 1.3969, + "theoretical_loss": 3.4810049017115983, + "tokens_seen": 1682767872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985050505050506, + "loss": 1.5008, + "theoretical_loss": 3.4809932110026303, + "tokens_seen": 1682833408 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.4591275751590729, + "objective/train/avg_token_score": 0.02467002533376217, + "objective/train/docs_used": 953335, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5312504768371582, + "objective/train/original_loss": 4.14010763168335, + "objective/train/theoretical_loss": 3.4809815208764077, + "objective/train/tokens_used": 41758176, + "objective/train/unlikelihood_loss": 0.059290263801813126, + "theoretical_loss": 3.4809815208764077, + "tokens_seen": 1682898944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984848484848485, + "loss": 1.423, + "theoretical_loss": 3.4809815208764077, + "tokens_seen": 1682898944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984646464646465, + "loss": 1.6025, + "theoretical_loss": 3.480969831332878, + "tokens_seen": 1682964480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984444444444445, + "loss": 1.6665, + "theoretical_loss": 3.4809581423719904, + "tokens_seen": 1683030016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984242424242425, + "loss": 1.472, + "theoretical_loss": 3.480946453993693, + "tokens_seen": 1683095552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984040404040404, + "loss": 1.4715, + "theoretical_loss": 3.480934766197934, + "tokens_seen": 1683161088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983838383838384, + "loss": 1.4211, + "theoretical_loss": 3.480923078984662, + "tokens_seen": 1683226624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983636363636363, + "loss": 1.495, + "theoretical_loss": 3.480911392353825, + "tokens_seen": 1683292160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983434343434344, + "loss": 1.2509, + "theoretical_loss": 3.4808997063053706, + "tokens_seen": 1683357696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983232323232323, + "loss": 1.3817, + "theoretical_loss": 3.480888020839248, + "tokens_seen": 1683423232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983030303030303, + "loss": 1.2817, + "theoretical_loss": 3.480876335955406, + "tokens_seen": 1683488768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982828282828282, + "loss": 1.4437, + "theoretical_loss": 3.4808646516537918, + "tokens_seen": 1683554304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982626262626263, + "loss": 1.5573, + "theoretical_loss": 3.4808529679343536, + "tokens_seen": 1683619840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982424242424243, + "loss": 1.6952, + "theoretical_loss": 3.480841284797041, + "tokens_seen": 1683685376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982222222222223, + "loss": 1.2028, + "theoretical_loss": 3.4808296022418013, + "tokens_seen": 1683750912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982020202020202, + "loss": 1.3336, + "theoretical_loss": 3.480817920268583, + "tokens_seen": 1683816448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981818181818182, + "loss": 1.3224, + "theoretical_loss": 3.480806238877335, + "tokens_seen": 1683881984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981616161616162, + "loss": 1.392, + "theoretical_loss": 3.4807945580680055, + "tokens_seen": 1683947520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981414141414142, + "loss": 1.3183, + "theoretical_loss": 3.480782877840542, + "tokens_seen": 1684013056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981212121212121, + "loss": 1.5275, + "theoretical_loss": 3.480771198194894, + "tokens_seen": 1684078592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981010101010101, + "loss": 1.3843, + "theoretical_loss": 3.4807595191310092, + "tokens_seen": 1684144128 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498080808080808, + "loss": 1.4248, + "theoretical_loss": 3.4807478406488364, + "tokens_seen": 1684209664 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498060606060606, + "loss": 1.5212, + "theoretical_loss": 3.4807361627483235, + "tokens_seen": 1684275200 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498040404040404, + "loss": 1.5897, + "theoretical_loss": 3.4807244854294197, + "tokens_seen": 1684340736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980202020202021, + "loss": 1.4325, + "theoretical_loss": 3.480712808692072, + "tokens_seen": 1684406272 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498, + "loss": 1.4759, + "theoretical_loss": 3.4807011325362307, + "tokens_seen": 1684471808 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.46138808131217957, + "objective/train/avg_token_score": 0.026906196027994156, + "objective/train/docs_used": 954676, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2592566013336182, + "objective/train/original_loss": 3.659281015396118, + "objective/train/theoretical_loss": 3.4806894569618425, + "objective/train/tokens_used": 43396576, + "objective/train/unlikelihood_loss": 0.058182328939437866, + "theoretical_loss": 3.4806894569618425, + "tokens_seen": 1684537344 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497979797979798, + "loss": 1.2782, + "theoretical_loss": 3.4806894569618425, + "tokens_seen": 1684537344 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497959595959596, + "loss": 1.3906, + "theoretical_loss": 3.480677781968857, + "tokens_seen": 1684602880 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497939393939394, + "loss": 1.5195, + "theoretical_loss": 3.480666107557221, + "tokens_seen": 1684668416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979191919191919, + "loss": 1.494, + "theoretical_loss": 3.4806544337268854, + "tokens_seen": 1684733952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978989898989899, + "loss": 1.5243, + "theoretical_loss": 3.4806427604777967, + "tokens_seen": 1684799488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978787878787879, + "loss": 1.3817, + "theoretical_loss": 3.4806310878099045, + "tokens_seen": 1684865024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978585858585858, + "loss": 1.3818, + "theoretical_loss": 3.4806194157231567, + "tokens_seen": 1684930560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978383838383838, + "loss": 1.3979, + "theoretical_loss": 3.480607744217502, + "tokens_seen": 1684996096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978181818181818, + "loss": 1.5915, + "theoretical_loss": 3.4805960732928876, + "tokens_seen": 1685061632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977979797979799, + "loss": 1.6397, + "theoretical_loss": 3.480584402949264, + "tokens_seen": 1685127168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977777777777778, + "loss": 1.7836, + "theoretical_loss": 3.4805727331865786, + "tokens_seen": 1685192704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977575757575758, + "loss": 1.4281, + "theoretical_loss": 3.48056106400478, + "tokens_seen": 1685258240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977373737373738, + "loss": 1.5813, + "theoretical_loss": 3.480549395403817, + "tokens_seen": 1685323776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977171717171717, + "loss": 1.5063, + "theoretical_loss": 3.4805377273836378, + "tokens_seen": 1685389312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976969696969697, + "loss": 1.3745, + "theoretical_loss": 3.4805260599441903, + "tokens_seen": 1685454848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976767676767677, + "loss": 1.5523, + "theoretical_loss": 3.480514393085424, + "tokens_seen": 1685520384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976565656565656, + "loss": 1.3951, + "theoretical_loss": 3.4805027268072877, + "tokens_seen": 1685585920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976363636363636, + "loss": 1.3416, + "theoretical_loss": 3.4804910611097286, + "tokens_seen": 1685651456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976161616161616, + "loss": 1.5686, + "theoretical_loss": 3.4804793959926963, + "tokens_seen": 1685716992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975959595959596, + "loss": 1.489, + "theoretical_loss": 3.480467731456139, + "tokens_seen": 1685782528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975757575757575, + "loss": 1.341, + "theoretical_loss": 3.4804560675000054, + "tokens_seen": 1685848064 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975555555555556, + "loss": 1.2872, + "theoretical_loss": 3.4804444041242437, + "tokens_seen": 1685913600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975353535353536, + "loss": 1.5454, + "theoretical_loss": 3.480432741328803, + "tokens_seen": 1685979136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975151515151516, + "loss": 1.5041, + "theoretical_loss": 3.4804210791136314, + "tokens_seen": 1686044672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004974949494949495, + "loss": 1.5567, + "theoretical_loss": 3.4804094174786773, + "tokens_seen": 1686110208 + }, + { + "epoch": 0.01, + "objective/train/aligned_token_count": 0.5329912304878235, + "objective/train/avg_token_score": 0.01913226768374443, + "objective/train/docs_used": 955358, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6620068550109863, + "objective/train/original_loss": 3.886705160140991, + "objective/train/theoretical_loss": 3.4803977564238897, + "objective/train/tokens_used": 45034976, + "objective/train/unlikelihood_loss": 0.04157567396759987, + "theoretical_loss": 3.4803977564238897, + "tokens_seen": 1686175744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004974747474747475, + "loss": 1.5445, + "theoretical_loss": 3.4803977564238897, + "tokens_seen": 1686175744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974545454545454, + "loss": 1.4179, + "theoretical_loss": 3.4803860959492177, + "tokens_seen": 1686241280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974343434343435, + "loss": 1.4412, + "theoretical_loss": 3.4803744360546087, + "tokens_seen": 1686306816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974141414141414, + "loss": 1.5498, + "theoretical_loss": 3.4803627767400123, + "tokens_seen": 1686372352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973939393939394, + "loss": 1.4011, + "theoretical_loss": 3.480351118005376, + "tokens_seen": 1686437888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973737373737373, + "loss": 1.4917, + "theoretical_loss": 3.48033945985065, + "tokens_seen": 1686503424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973535353535353, + "loss": 1.392, + "theoretical_loss": 3.4803278022757818, + "tokens_seen": 1686568960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973333333333334, + "loss": 1.4304, + "theoretical_loss": 3.4803161452807196, + "tokens_seen": 1686634496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973131313131314, + "loss": 1.4172, + "theoretical_loss": 3.4803044888654133, + "tokens_seen": 1686700032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972929292929293, + "loss": 1.2836, + "theoretical_loss": 3.480292833029811, + "tokens_seen": 1686765568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972727272727273, + "loss": 1.5495, + "theoretical_loss": 3.480281177773861, + "tokens_seen": 1686831104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972525252525253, + "loss": 1.3092, + "theoretical_loss": 3.4802695230975127, + "tokens_seen": 1686896640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972323232323233, + "loss": 1.3053, + "theoretical_loss": 3.480257869000714, + "tokens_seen": 1686962176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972121212121212, + "loss": 1.3317, + "theoretical_loss": 3.4802462154834135, + "tokens_seen": 1687027712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971919191919192, + "loss": 1.484, + "theoretical_loss": 3.4802345625455606, + "tokens_seen": 1687093248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971717171717171, + "loss": 1.4059, + "theoretical_loss": 3.480222910187104, + "tokens_seen": 1687158784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971515151515152, + "loss": 1.5274, + "theoretical_loss": 3.4802112584079916, + "tokens_seen": 1687224320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971313131313131, + "loss": 1.1674, + "theoretical_loss": 3.4801996072081725, + "tokens_seen": 1687289856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971111111111111, + "loss": 1.721, + "theoretical_loss": 3.4801879565875957, + "tokens_seen": 1687355392 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497090909090909, + "loss": 1.2672, + "theoretical_loss": 3.4801763065462095, + "tokens_seen": 1687420928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970707070707071, + "loss": 1.6776, + "theoretical_loss": 3.4801646570839626, + "tokens_seen": 1687486464 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970505050505051, + "loss": 1.3374, + "theoretical_loss": 3.480153008200804, + "tokens_seen": 1687552000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970303030303031, + "loss": 1.2859, + "theoretical_loss": 3.480141359896683, + "tokens_seen": 1687617536 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497010101010101, + "loss": 1.5063, + "theoretical_loss": 3.4801297121715464, + "tokens_seen": 1687683072 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496989898989899, + "loss": 1.3917, + "theoretical_loss": 3.4801180650253447, + "tokens_seen": 1687748608 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.4268084168434143, + "objective/train/avg_token_score": 0.010676675476133823, + "objective/train/docs_used": 956671, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4835927486419678, + "objective/train/original_loss": 4.212736129760742, + "objective/train/theoretical_loss": 3.480106418458026, + "objective/train/tokens_used": 46673376, + "objective/train/unlikelihood_loss": 0.08470705896615982, + "theoretical_loss": 3.480106418458026, + "tokens_seen": 1687814144 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496969696969697, + "loss": 1.4357, + "theoretical_loss": 3.480106418458026, + "tokens_seen": 1687814144 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496949494949495, + "loss": 1.3479, + "theoretical_loss": 3.4800947724695392, + "tokens_seen": 1687879680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969292929292929, + "loss": 1.2908, + "theoretical_loss": 3.4800831270598334, + "tokens_seen": 1687945216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969090909090909, + "loss": 1.506, + "theoretical_loss": 3.4800714822288565, + "tokens_seen": 1688010752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968888888888889, + "loss": 1.5604, + "theoretical_loss": 3.4800598379765577, + "tokens_seen": 1688076288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968686868686868, + "loss": 1.435, + "theoretical_loss": 3.4800481943028863, + "tokens_seen": 1688141824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968484848484849, + "loss": 1.3855, + "theoretical_loss": 3.48003655120779, + "tokens_seen": 1688207360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968282828282829, + "loss": 1.579, + "theoretical_loss": 3.4800249086912185, + "tokens_seen": 1688272896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968080808080808, + "loss": 1.5069, + "theoretical_loss": 3.4800132667531205, + "tokens_seen": 1688338432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967878787878788, + "loss": 1.5546, + "theoretical_loss": 3.4800016253934447, + "tokens_seen": 1688403968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967676767676768, + "loss": 1.2675, + "theoretical_loss": 3.4799899846121396, + "tokens_seen": 1688469504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967474747474748, + "loss": 1.5451, + "theoretical_loss": 3.4799783444091545, + "tokens_seen": 1688535040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967272727272727, + "loss": 1.4007, + "theoretical_loss": 3.4799667047844376, + "tokens_seen": 1688600576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967070707070707, + "loss": 1.4922, + "theoretical_loss": 3.479955065737938, + "tokens_seen": 1688666112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966868686868687, + "loss": 1.6596, + "theoretical_loss": 3.479943427269605, + "tokens_seen": 1688731648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966666666666666, + "loss": 1.7516, + "theoretical_loss": 3.4799317893793873, + "tokens_seen": 1688797184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966464646464646, + "loss": 1.437, + "theoretical_loss": 3.479920152067233, + "tokens_seen": 1688862720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966262626262627, + "loss": 1.3793, + "theoretical_loss": 3.4799085153330918, + "tokens_seen": 1688928256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966060606060607, + "loss": 1.6005, + "theoretical_loss": 3.479896879176912, + "tokens_seen": 1688993792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965858585858586, + "loss": 1.5023, + "theoretical_loss": 3.479885243598643, + "tokens_seen": 1689059328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965656565656566, + "loss": 1.4687, + "theoretical_loss": 3.4798736085982336, + "tokens_seen": 1689124864 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965454545454546, + "loss": 1.6716, + "theoretical_loss": 3.479861974175632, + "tokens_seen": 1689190400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965252525252525, + "loss": 1.6124, + "theoretical_loss": 3.4798503403307874, + "tokens_seen": 1689255936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965050505050505, + "loss": 1.2172, + "theoretical_loss": 3.4798387070636494, + "tokens_seen": 1689321472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964848484848485, + "loss": 1.7067, + "theoretical_loss": 3.479827074374166, + "tokens_seen": 1689387008 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.4199046790599823, + "objective/train/avg_token_score": 0.011589493602514267, + "objective/train/docs_used": 957370, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3580185174942017, + "objective/train/original_loss": 4.241112232208252, + "objective/train/theoretical_loss": 3.4798154422622867, + "objective/train/tokens_used": 48311776, + "objective/train/unlikelihood_loss": 0.06464245170354843, + "theoretical_loss": 3.4798154422622867, + "tokens_seen": 1689452544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964646464646464, + "loss": 1.4656, + "theoretical_loss": 3.4798154422622867, + "tokens_seen": 1689452544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964444444444444, + "loss": 1.4522, + "theoretical_loss": 3.4798038107279603, + "tokens_seen": 1689518080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964242424242424, + "loss": 1.5003, + "theoretical_loss": 3.4797921797711355, + "tokens_seen": 1689583616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964040404040405, + "loss": 1.4859, + "theoretical_loss": 3.4797805493917613, + "tokens_seen": 1689649152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963838383838384, + "loss": 1.5211, + "theoretical_loss": 3.4797689195897865, + "tokens_seen": 1689714688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963636363636364, + "loss": 1.5367, + "theoretical_loss": 3.4797572903651606, + "tokens_seen": 1689780224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963434343434344, + "loss": 1.4631, + "theoretical_loss": 3.479745661717832, + "tokens_seen": 1689845760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963232323232324, + "loss": 1.4319, + "theoretical_loss": 3.47973403364775, + "tokens_seen": 1689911296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963030303030303, + "loss": 1.2408, + "theoretical_loss": 3.4797224061548633, + "tokens_seen": 1689976832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962828282828283, + "loss": 1.3776, + "theoretical_loss": 3.4797107792391206, + "tokens_seen": 1690042368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962626262626262, + "loss": 1.4295, + "theoretical_loss": 3.4796991529004715, + "tokens_seen": 1690107904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962424242424243, + "loss": 1.5602, + "theoretical_loss": 3.4796875271388648, + "tokens_seen": 1690173440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962222222222222, + "loss": 1.3299, + "theoretical_loss": 3.4796759019542494, + "tokens_seen": 1690238976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962020202020202, + "loss": 1.4424, + "theoretical_loss": 3.479664277346574, + "tokens_seen": 1690304512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961818181818181, + "loss": 1.6119, + "theoretical_loss": 3.479652653315788, + "tokens_seen": 1690370048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961616161616161, + "loss": 1.5685, + "theoretical_loss": 3.4796410298618405, + "tokens_seen": 1690435584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961414141414142, + "loss": 1.4242, + "theoretical_loss": 3.4796294069846803, + "tokens_seen": 1690501120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961212121212122, + "loss": 1.4835, + "theoretical_loss": 3.4796177846842564, + "tokens_seen": 1690566656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961010101010101, + "loss": 1.3436, + "theoretical_loss": 3.479606162960518, + "tokens_seen": 1690632192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960808080808081, + "loss": 1.6164, + "theoretical_loss": 3.4795945418134133, + "tokens_seen": 1690697728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960606060606061, + "loss": 1.6758, + "theoretical_loss": 3.479582921242893, + "tokens_seen": 1690763264 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960404040404041, + "loss": 1.298, + "theoretical_loss": 3.4795713012489045, + "tokens_seen": 1690828800 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496020202020202, + "loss": 1.4461, + "theoretical_loss": 3.4795596818313976, + "tokens_seen": 1690894336 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496, + "loss": 1.4362, + "theoretical_loss": 3.4795480629903217, + "tokens_seen": 1690959872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959797979797979, + "loss": 1.3436, + "theoretical_loss": 3.479536444725625, + "tokens_seen": 1691025408 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.38599705696105957, + "objective/train/avg_token_score": 0.048682406544685364, + "objective/train/docs_used": 958661, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2436673641204834, + "objective/train/original_loss": 4.153236389160156, + "objective/train/theoretical_loss": 3.4795248270372574, + "objective/train/tokens_used": 49950176, + "objective/train/unlikelihood_loss": 0.07140035182237625, + "theoretical_loss": 3.4795248270372574, + "tokens_seen": 1691090944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959595959595959, + "loss": 1.2475, + "theoretical_loss": 3.4795248270372574, + "tokens_seen": 1691090944 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495939393939394, + "loss": 1.6063, + "theoretical_loss": 3.479513209925167, + "tokens_seen": 1691156480 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495919191919192, + "loss": 1.4383, + "theoretical_loss": 3.479501593389304, + "tokens_seen": 1691222016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958989898989899, + "loss": 1.6435, + "theoretical_loss": 3.479489977429617, + "tokens_seen": 1691287552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958787878787879, + "loss": 1.5405, + "theoretical_loss": 3.4794783620460548, + "tokens_seen": 1691353088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958585858585859, + "loss": 1.5615, + "theoretical_loss": 3.479466747238567, + "tokens_seen": 1691418624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958383838383839, + "loss": 1.51, + "theoretical_loss": 3.4794551330071024, + "tokens_seen": 1691484160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958181818181818, + "loss": 1.4175, + "theoretical_loss": 3.47944351935161, + "tokens_seen": 1691549696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957979797979798, + "loss": 1.5496, + "theoretical_loss": 3.4794319062720396, + "tokens_seen": 1691615232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957777777777778, + "loss": 1.4502, + "theoretical_loss": 3.479420293768339, + "tokens_seen": 1691680768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957575757575757, + "loss": 1.4327, + "theoretical_loss": 3.479408681840459, + "tokens_seen": 1691746304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957373737373737, + "loss": 1.4366, + "theoretical_loss": 3.4793970704883477, + "tokens_seen": 1691811840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957171717171717, + "loss": 1.2897, + "theoretical_loss": 3.4793854597119545, + "tokens_seen": 1691877376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956969696969698, + "loss": 1.6798, + "theoretical_loss": 3.4793738495112283, + "tokens_seen": 1691942912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956767676767677, + "loss": 1.2665, + "theoretical_loss": 3.4793622398861186, + "tokens_seen": 1692008448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956565656565657, + "loss": 1.5467, + "theoretical_loss": 3.4793506308365747, + "tokens_seen": 1692073984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956363636363637, + "loss": 1.3853, + "theoretical_loss": 3.479339022362545, + "tokens_seen": 1692139520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956161616161616, + "loss": 1.5345, + "theoretical_loss": 3.47932741446398, + "tokens_seen": 1692205056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955959595959596, + "loss": 1.6217, + "theoretical_loss": 3.4793158071408277, + "tokens_seen": 1692270592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955757575757576, + "loss": 1.6179, + "theoretical_loss": 3.4793042003930377, + "tokens_seen": 1692336128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955555555555556, + "loss": 1.3264, + "theoretical_loss": 3.479292594220559, + "tokens_seen": 1692401664 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955353535353535, + "loss": 1.6683, + "theoretical_loss": 3.4792809886233416, + "tokens_seen": 1692467200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955151515151515, + "loss": 1.6922, + "theoretical_loss": 3.4792693836013333, + "tokens_seen": 1692532736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954949494949495, + "loss": 1.6503, + "theoretical_loss": 3.4792577791544845, + "tokens_seen": 1692598272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954747474747474, + "loss": 1.6034, + "theoretical_loss": 3.4792461752827446, + "tokens_seen": 1692663808 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.4952957034111023, + "objective/train/avg_token_score": 0.008989947848021984, + "objective/train/docs_used": 959159, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6178122758865356, + "objective/train/original_loss": 3.7790725231170654, + "objective/train/theoretical_loss": 3.479234571986062, + "objective/train/tokens_used": 51588576, + "objective/train/unlikelihood_loss": 0.05506987124681473, + "theoretical_loss": 3.479234571986062, + "tokens_seen": 1692729344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954545454545455, + "loss": 1.5392, + "theoretical_loss": 3.479234571986062, + "tokens_seen": 1692729344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954343434343435, + "loss": 1.5108, + "theoretical_loss": 3.479222969264386, + "tokens_seen": 1692794880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954141414141415, + "loss": 1.404, + "theoretical_loss": 3.4792113671176663, + "tokens_seen": 1692860416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953939393939394, + "loss": 1.4155, + "theoretical_loss": 3.479199765545852, + "tokens_seen": 1692925952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953737373737374, + "loss": 1.4939, + "theoretical_loss": 3.4791881645488916, + "tokens_seen": 1692991488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953535353535354, + "loss": 1.5546, + "theoretical_loss": 3.479176564126736, + "tokens_seen": 1693057024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953333333333334, + "loss": 1.6032, + "theoretical_loss": 3.4791649642793328, + "tokens_seen": 1693122560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953131313131313, + "loss": 1.4701, + "theoretical_loss": 3.479153365006632, + "tokens_seen": 1693188096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952929292929293, + "loss": 1.4542, + "theoretical_loss": 3.479141766308584, + "tokens_seen": 1693253632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952727272727272, + "loss": 1.508, + "theoretical_loss": 3.479130168185136, + "tokens_seen": 1693319168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952525252525252, + "loss": 1.5405, + "theoretical_loss": 3.4791185706362384, + "tokens_seen": 1693384704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952323232323232, + "loss": 1.3952, + "theoretical_loss": 3.4791069736618407, + "tokens_seen": 1693450240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952121212121213, + "loss": 1.4216, + "theoretical_loss": 3.479095377261892, + "tokens_seen": 1693515776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951919191919192, + "loss": 1.5664, + "theoretical_loss": 3.4790837814363407, + "tokens_seen": 1693581312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951717171717172, + "loss": 1.6256, + "theoretical_loss": 3.479072186185138, + "tokens_seen": 1693646848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951515151515152, + "loss": 1.3279, + "theoretical_loss": 3.4790605915082313, + "tokens_seen": 1693712384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951313131313132, + "loss": 1.3493, + "theoretical_loss": 3.479048997405571, + "tokens_seen": 1693777920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951111111111111, + "loss": 1.6102, + "theoretical_loss": 3.479037403877107, + "tokens_seen": 1693843456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950909090909091, + "loss": 1.2949, + "theoretical_loss": 3.4790258109227867, + "tokens_seen": 1693908992 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495070707070707, + "loss": 1.3058, + "theoretical_loss": 3.479014218542561, + "tokens_seen": 1693974528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950505050505051, + "loss": 1.375, + "theoretical_loss": 3.4790026267363796, + "tokens_seen": 1694040064 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495030303030303, + "loss": 1.4453, + "theoretical_loss": 3.478991035504191, + "tokens_seen": 1694105600 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495010101010101, + "loss": 1.55, + "theoretical_loss": 3.4789794448459443, + "tokens_seen": 1694171136 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494989898989899, + "loss": 1.5699, + "theoretical_loss": 3.4789678547615894, + "tokens_seen": 1694236672 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494969696969697, + "loss": 1.3551, + "theoretical_loss": 3.478956265251076, + "tokens_seen": 1694302208 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.3170210123062134, + "objective/train/avg_token_score": 0.017827244475483894, + "objective/train/docs_used": 960536, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9088751673698425, + "objective/train/original_loss": 4.046257972717285, + "objective/train/theoretical_loss": 3.478944676314353, + "objective/train/tokens_used": 53226976, + "objective/train/unlikelihood_loss": 0.10128523409366608, + "theoretical_loss": 3.478944676314353, + "tokens_seen": 1694367744 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494949494949495, + "loss": 1.3764, + "theoretical_loss": 3.478944676314353, + "tokens_seen": 1694367744 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494929292929293, + "loss": 1.7527, + "theoretical_loss": 3.4789330879513694, + "tokens_seen": 1694433280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949090909090909, + "loss": 1.5846, + "theoretical_loss": 3.478921500162076, + "tokens_seen": 1694498816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948888888888889, + "loss": 1.3229, + "theoretical_loss": 3.4789099129464205, + "tokens_seen": 1694564352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948686868686869, + "loss": 1.5288, + "theoretical_loss": 3.4788983263043534, + "tokens_seen": 1694629888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948484848484849, + "loss": 1.5913, + "theoretical_loss": 3.478886740235824, + "tokens_seen": 1694695424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948282828282828, + "loss": 1.4224, + "theoretical_loss": 3.4788751547407815, + "tokens_seen": 1694760960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948080808080808, + "loss": 1.6395, + "theoretical_loss": 3.478863569819176, + "tokens_seen": 1694826496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947878787878787, + "loss": 1.4925, + "theoretical_loss": 3.478851985470956, + "tokens_seen": 1694892032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947676767676767, + "loss": 1.5345, + "theoretical_loss": 3.478840401696071, + "tokens_seen": 1694957568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947474747474748, + "loss": 1.3046, + "theoretical_loss": 3.4788288184944713, + "tokens_seen": 1695023104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947272727272728, + "loss": 1.5731, + "theoretical_loss": 3.4788172358661056, + "tokens_seen": 1695088640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947070707070707, + "loss": 1.3865, + "theoretical_loss": 3.478805653810924, + "tokens_seen": 1695154176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946868686868687, + "loss": 1.3987, + "theoretical_loss": 3.478794072328875, + "tokens_seen": 1695219712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946666666666667, + "loss": 1.4943, + "theoretical_loss": 3.478782491419909, + "tokens_seen": 1695285248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946464646464647, + "loss": 1.2217, + "theoretical_loss": 3.4787709110839753, + "tokens_seen": 1695350784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946262626262626, + "loss": 1.4646, + "theoretical_loss": 3.478759331321023, + "tokens_seen": 1695416320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946060606060606, + "loss": 1.4741, + "theoretical_loss": 3.478747752131002, + "tokens_seen": 1695481856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945858585858586, + "loss": 1.4813, + "theoretical_loss": 3.4787361735138624, + "tokens_seen": 1695547392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945656565656565, + "loss": 1.3267, + "theoretical_loss": 3.478724595469552, + "tokens_seen": 1695612928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945454545454545, + "loss": 1.3645, + "theoretical_loss": 3.4787130179980217, + "tokens_seen": 1695678464 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945252525252526, + "loss": 1.4331, + "theoretical_loss": 3.478701441099221, + "tokens_seen": 1695744000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945050505050506, + "loss": 1.481, + "theoretical_loss": 3.4786898647730986, + "tokens_seen": 1695809536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944848484848485, + "loss": 1.4608, + "theoretical_loss": 3.478678289019605, + "tokens_seen": 1695875072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944646464646465, + "loss": 1.4611, + "theoretical_loss": 3.478666713838689, + "tokens_seen": 1695940608 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.3928396999835968, + "objective/train/avg_token_score": 0.021624328568577766, + "objective/train/docs_used": 961177, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3411294221878052, + "objective/train/original_loss": 4.00459623336792, + "objective/train/theoretical_loss": 3.478655139230301, + "objective/train/tokens_used": 54865376, + "objective/train/unlikelihood_loss": 0.05852148309350014, + "theoretical_loss": 3.478655139230301, + "tokens_seen": 1696006144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944444444444445, + "loss": 1.4319, + "theoretical_loss": 3.478655139230301, + "tokens_seen": 1696006144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944242424242424, + "loss": 1.432, + "theoretical_loss": 3.478643565194389, + "tokens_seen": 1696071680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944040404040404, + "loss": 1.4987, + "theoretical_loss": 3.478631991730904, + "tokens_seen": 1696137216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943838383838384, + "loss": 1.6415, + "theoretical_loss": 3.478620418839795, + "tokens_seen": 1696202752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943636363636363, + "loss": 1.6562, + "theoretical_loss": 3.4786088465210123, + "tokens_seen": 1696268288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943434343434343, + "loss": 1.3873, + "theoretical_loss": 3.478597274774504, + "tokens_seen": 1696333824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943232323232323, + "loss": 1.3246, + "theoretical_loss": 3.4785857036002215, + "tokens_seen": 1696399360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943030303030303, + "loss": 1.3636, + "theoretical_loss": 3.478574132998113, + "tokens_seen": 1696464896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942828282828283, + "loss": 1.446, + "theoretical_loss": 3.478562562968129, + "tokens_seen": 1696530432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942626262626263, + "loss": 1.3962, + "theoretical_loss": 3.4785509935102183, + "tokens_seen": 1696595968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942424242424243, + "loss": 1.4457, + "theoretical_loss": 3.4785394246243317, + "tokens_seen": 1696661504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942222222222223, + "loss": 1.6089, + "theoretical_loss": 3.478527856310417, + "tokens_seen": 1696727040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942020202020202, + "loss": 1.5574, + "theoretical_loss": 3.4785162885684255, + "tokens_seen": 1696792576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941818181818182, + "loss": 1.4105, + "theoretical_loss": 3.478504721398306, + "tokens_seen": 1696858112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941616161616161, + "loss": 1.399, + "theoretical_loss": 3.4784931548000086, + "tokens_seen": 1696923648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941414141414142, + "loss": 1.5323, + "theoretical_loss": 3.4784815887734823, + "tokens_seen": 1696989184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941212121212121, + "loss": 1.5829, + "theoretical_loss": 3.4784700233186774, + "tokens_seen": 1697054720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941010101010101, + "loss": 1.4786, + "theoretical_loss": 3.4784584584355436, + "tokens_seen": 1697120256 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494080808080808, + "loss": 1.5934, + "theoretical_loss": 3.47844689412403, + "tokens_seen": 1697185792 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494060606060606, + "loss": 1.4969, + "theoretical_loss": 3.4784353303840865, + "tokens_seen": 1697251328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940404040404041, + "loss": 1.5708, + "theoretical_loss": 3.478423767215663, + "tokens_seen": 1697316864 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940202020202021, + "loss": 1.3845, + "theoretical_loss": 3.478412204618709, + "tokens_seen": 1697382400 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494, + "loss": 1.3359, + "theoretical_loss": 3.4784006425931744, + "tokens_seen": 1697447936 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493979797979798, + "loss": 1.4839, + "theoretical_loss": 3.478389081139009, + "tokens_seen": 1697513472 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493959595959596, + "loss": 1.5483, + "theoretical_loss": 3.4783775202561618, + "tokens_seen": 1697579008 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.5273093581199646, + "objective/train/avg_token_score": 0.005273816175758839, + "objective/train/docs_used": 961828, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.706133484840393, + "objective/train/original_loss": 3.877432107925415, + "objective/train/theoretical_loss": 3.4783659599445826, + "objective/train/tokens_used": 56503776, + "objective/train/unlikelihood_loss": 0.0712694600224495, + "theoretical_loss": 3.4783659599445826, + "tokens_seen": 1697644544 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493939393939394, + "loss": 1.5571, + "theoretical_loss": 3.4783659599445826, + "tokens_seen": 1697644544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939191919191919, + "loss": 1.2926, + "theoretical_loss": 3.478354400204222, + "tokens_seen": 1697710080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938989898989899, + "loss": 1.3542, + "theoretical_loss": 3.4783428410350297, + "tokens_seen": 1697775616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938787878787878, + "loss": 1.6162, + "theoretical_loss": 3.4783312824369537, + "tokens_seen": 1697841152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938585858585859, + "loss": 1.5232, + "theoretical_loss": 3.478319724409946, + "tokens_seen": 1697906688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938383838383838, + "loss": 1.5115, + "theoretical_loss": 3.478308166953955, + "tokens_seen": 1697972224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938181818181819, + "loss": 1.6203, + "theoretical_loss": 3.4782966100689308, + "tokens_seen": 1698037760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937979797979798, + "loss": 1.6335, + "theoretical_loss": 3.4782850537548233, + "tokens_seen": 1698103296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937777777777778, + "loss": 1.3283, + "theoretical_loss": 3.478273498011582, + "tokens_seen": 1698168832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937575757575758, + "loss": 1.5045, + "theoretical_loss": 3.4782619428391564, + "tokens_seen": 1698234368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937373737373738, + "loss": 1.5789, + "theoretical_loss": 3.478250388237497, + "tokens_seen": 1698299904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937171717171717, + "loss": 1.4186, + "theoretical_loss": 3.478238834206553, + "tokens_seen": 1698365440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936969696969697, + "loss": 1.4197, + "theoretical_loss": 3.4782272807462746, + "tokens_seen": 1698430976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936767676767677, + "loss": 1.384, + "theoretical_loss": 3.4782157278566115, + "tokens_seen": 1698496512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936565656565657, + "loss": 1.553, + "theoretical_loss": 3.4782041755375133, + "tokens_seen": 1698562048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936363636363636, + "loss": 1.4178, + "theoretical_loss": 3.47819262378893, + "tokens_seen": 1698627584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936161616161616, + "loss": 1.6003, + "theoretical_loss": 3.478181072610811, + "tokens_seen": 1698693120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935959595959597, + "loss": 1.0584, + "theoretical_loss": 3.478169522003107, + "tokens_seen": 1698758656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935757575757576, + "loss": 1.4205, + "theoretical_loss": 3.478157971965767, + "tokens_seen": 1698824192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935555555555556, + "loss": 1.4223, + "theoretical_loss": 3.4781464224987415, + "tokens_seen": 1698889728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935353535353536, + "loss": 1.3599, + "theoretical_loss": 3.478134873601979, + "tokens_seen": 1698955264 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935151515151515, + "loss": 1.375, + "theoretical_loss": 3.4781233252754316, + "tokens_seen": 1699020800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934949494949495, + "loss": 1.4275, + "theoretical_loss": 3.4781117775190467, + "tokens_seen": 1699086336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934747474747475, + "loss": 1.3522, + "theoretical_loss": 3.4781002303327764, + "tokens_seen": 1699151872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934545454545455, + "loss": 1.48, + "theoretical_loss": 3.4780886837165688, + "tokens_seen": 1699217408 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.35807672142982483, + "objective/train/avg_token_score": 0.012293923646211624, + "objective/train/docs_used": 963095, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1824865341186523, + "objective/train/original_loss": 4.2231035232543945, + "objective/train/theoretical_loss": 3.4780771376703745, + "objective/train/tokens_used": 58142176, + "objective/train/unlikelihood_loss": 0.0799039676785469, + "theoretical_loss": 3.4780771376703745, + "tokens_seen": 1699282944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934343434343434, + "loss": 1.2993, + "theoretical_loss": 3.4780771376703745, + "tokens_seen": 1699282944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934141414141414, + "loss": 1.4704, + "theoretical_loss": 3.4780655921941435, + "tokens_seen": 1699348480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933939393939394, + "loss": 1.3564, + "theoretical_loss": 3.4780540472878254, + "tokens_seen": 1699414016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933737373737373, + "loss": 1.487, + "theoretical_loss": 3.47804250295137, + "tokens_seen": 1699479552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933535353535354, + "loss": 1.331, + "theoretical_loss": 3.478030959184728, + "tokens_seen": 1699545088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933333333333334, + "loss": 1.6139, + "theoretical_loss": 3.4780194159878484, + "tokens_seen": 1699610624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933131313131314, + "loss": 1.4798, + "theoretical_loss": 3.4780078733606814, + "tokens_seen": 1699676160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932929292929293, + "loss": 1.516, + "theoretical_loss": 3.477996331303177, + "tokens_seen": 1699741696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932727272727273, + "loss": 1.5999, + "theoretical_loss": 3.4779847898152854, + "tokens_seen": 1699807232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932525252525253, + "loss": 1.4377, + "theoretical_loss": 3.477973248896956, + "tokens_seen": 1699872768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932323232323232, + "loss": 1.4555, + "theoretical_loss": 3.4779617085481385, + "tokens_seen": 1699938304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932121212121212, + "loss": 1.2517, + "theoretical_loss": 3.4779501687687837, + "tokens_seen": 1700003840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931919191919192, + "loss": 1.5707, + "theoretical_loss": 3.477938629558841, + "tokens_seen": 1700069376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931717171717171, + "loss": 1.4201, + "theoretical_loss": 3.4779270909182607, + "tokens_seen": 1700134912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931515151515151, + "loss": 1.539, + "theoretical_loss": 3.4779155528469925, + "tokens_seen": 1700200448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931313131313131, + "loss": 1.4055, + "theoretical_loss": 3.477904015344986, + "tokens_seen": 1700265984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931111111111112, + "loss": 1.628, + "theoretical_loss": 3.4778924784121914, + "tokens_seen": 1700331520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930909090909091, + "loss": 1.4366, + "theoretical_loss": 3.47788094204856, + "tokens_seen": 1700397056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930707070707071, + "loss": 1.5349, + "theoretical_loss": 3.4778694062540394, + "tokens_seen": 1700462592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930505050505051, + "loss": 1.335, + "theoretical_loss": 3.4778578710285815, + "tokens_seen": 1700528128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930303030303031, + "loss": 1.4479, + "theoretical_loss": 3.477846336372136, + "tokens_seen": 1700593664 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493010101010101, + "loss": 1.6763, + "theoretical_loss": 3.4778348022846517, + "tokens_seen": 1700659200 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492989898989899, + "loss": 1.5868, + "theoretical_loss": 3.47782326876608, + "tokens_seen": 1700724736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929696969696969, + "loss": 1.4506, + "theoretical_loss": 3.4778117358163705, + "tokens_seen": 1700790272 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492949494949495, + "loss": 1.5949, + "theoretical_loss": 3.4778002034354722, + "tokens_seen": 1700855808 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.4352394938468933, + "objective/train/avg_token_score": 0.022302182391285896, + "objective/train/docs_used": 963811, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3806675672531128, + "objective/train/original_loss": 3.8218584060668945, + "objective/train/theoretical_loss": 3.477788671623337, + "objective/train/tokens_used": 59780576, + "objective/train/unlikelihood_loss": 0.0842813178896904, + "theoretical_loss": 3.477788671623337, + "tokens_seen": 1700921344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929292929292929, + "loss": 1.6276, + "theoretical_loss": 3.477788671623337, + "tokens_seen": 1700921344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929090909090909, + "loss": 1.3813, + "theoretical_loss": 3.4777771403799136, + "tokens_seen": 1700986880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928888888888888, + "loss": 1.6226, + "theoretical_loss": 3.4777656097051524, + "tokens_seen": 1701052416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928686868686869, + "loss": 1.6444, + "theoretical_loss": 3.4777540795990034, + "tokens_seen": 1701117952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928484848484849, + "loss": 1.3509, + "theoretical_loss": 3.477742550061417, + "tokens_seen": 1701183488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928282828282829, + "loss": 1.2636, + "theoretical_loss": 3.4777310210923424, + "tokens_seen": 1701249024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928080808080808, + "loss": 1.486, + "theoretical_loss": 3.4777194926917305, + "tokens_seen": 1701314560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927878787878788, + "loss": 1.339, + "theoretical_loss": 3.4777079648595315, + "tokens_seen": 1701380096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927676767676768, + "loss": 1.5191, + "theoretical_loss": 3.4776964375956947, + "tokens_seen": 1701445632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927474747474748, + "loss": 1.4387, + "theoretical_loss": 3.477684910900171, + "tokens_seen": 1701511168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927272727272727, + "loss": 1.3134, + "theoretical_loss": 3.4776733847729098, + "tokens_seen": 1701576704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927070707070707, + "loss": 1.4071, + "theoretical_loss": 3.4776618592138613, + "tokens_seen": 1701642240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926868686868686, + "loss": 1.3664, + "theoretical_loss": 3.477650334222976, + "tokens_seen": 1701707776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926666666666666, + "loss": 1.5606, + "theoretical_loss": 3.477638809800204, + "tokens_seen": 1701773312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926464646464647, + "loss": 1.4081, + "theoretical_loss": 3.477627285945495, + "tokens_seen": 1701838848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926262626262627, + "loss": 1.5566, + "theoretical_loss": 3.4776157626587993, + "tokens_seen": 1701904384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926060606060606, + "loss": 1.3377, + "theoretical_loss": 3.4776042399400673, + "tokens_seen": 1701969920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925858585858586, + "loss": 1.1917, + "theoretical_loss": 3.477592717789248, + "tokens_seen": 1702035456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925656565656566, + "loss": 1.3057, + "theoretical_loss": 3.4775811962062937, + "tokens_seen": 1702100992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925454545454546, + "loss": 1.4049, + "theoretical_loss": 3.4775696751911527, + "tokens_seen": 1702166528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925252525252525, + "loss": 1.5844, + "theoretical_loss": 3.4775581547437757, + "tokens_seen": 1702232064 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925050505050505, + "loss": 1.5742, + "theoretical_loss": 3.477546634864113, + "tokens_seen": 1702297600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924848484848485, + "loss": 1.6743, + "theoretical_loss": 3.477535115552115, + "tokens_seen": 1702363136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924646464646464, + "loss": 1.7692, + "theoretical_loss": 3.4775235968077314, + "tokens_seen": 1702428672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924444444444444, + "loss": 1.4189, + "theoretical_loss": 3.4775120786309124, + "tokens_seen": 1702494208 + }, + { + "epoch": 0.02, + "objective/train/aligned_token_count": 0.4039589464664459, + "objective/train/avg_token_score": 0.022879842668771744, + "objective/train/docs_used": 965136, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2613023519515991, + "objective/train/original_loss": 4.1900811195373535, + "objective/train/theoretical_loss": 3.4775005610216088, + "objective/train/tokens_used": 61418976, + "objective/train/unlikelihood_loss": 0.07926816493272781, + "theoretical_loss": 3.4775005610216088, + "tokens_seen": 1702559744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004924242424242425, + "loss": 1.3197, + "theoretical_loss": 3.4775005610216088, + "tokens_seen": 1702559744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004924040404040405, + "loss": 1.4524, + "theoretical_loss": 3.4774890439797694, + "tokens_seen": 1702625280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923838383838384, + "loss": 1.4616, + "theoretical_loss": 3.4774775275053464, + "tokens_seen": 1702690816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923636363636364, + "loss": 1.4173, + "theoretical_loss": 3.4774660115982883, + "tokens_seen": 1702756352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923434343434344, + "loss": 1.4959, + "theoretical_loss": 3.477454496258546, + "tokens_seen": 1702821888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923232323232323, + "loss": 1.4659, + "theoretical_loss": 3.47744298148607, + "tokens_seen": 1702887424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923030303030303, + "loss": 1.4347, + "theoretical_loss": 3.4774314672808098, + "tokens_seen": 1702952960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922828282828283, + "loss": 1.4188, + "theoretical_loss": 3.477419953642716, + "tokens_seen": 1703018496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922626262626262, + "loss": 1.4657, + "theoretical_loss": 3.477408440571739, + "tokens_seen": 1703084032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922424242424242, + "loss": 1.2297, + "theoretical_loss": 3.4773969280678294, + "tokens_seen": 1703149568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922222222222222, + "loss": 1.2242, + "theoretical_loss": 3.4773854161309363, + "tokens_seen": 1703215104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922020202020202, + "loss": 1.3182, + "theoretical_loss": 3.477373904761011, + "tokens_seen": 1703280640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921818181818182, + "loss": 1.4721, + "theoretical_loss": 3.4773623939580034, + "tokens_seen": 1703346176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921616161616162, + "loss": 1.7844, + "theoretical_loss": 3.477350883721863, + "tokens_seen": 1703411712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921414141414142, + "loss": 1.3457, + "theoretical_loss": 3.477339374052542, + "tokens_seen": 1703477248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921212121212122, + "loss": 1.3502, + "theoretical_loss": 3.477327864949989, + "tokens_seen": 1703542784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921010101010101, + "loss": 1.6302, + "theoretical_loss": 3.4773163564141547, + "tokens_seen": 1703608320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920808080808081, + "loss": 1.3801, + "theoretical_loss": 3.4773048484449895, + "tokens_seen": 1703673856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920606060606061, + "loss": 1.4358, + "theoretical_loss": 3.4772933410424436, + "tokens_seen": 1703739392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920404040404041, + "loss": 1.7846, + "theoretical_loss": 3.477281834206468, + "tokens_seen": 1703804928 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492020202020202, + "loss": 1.4055, + "theoretical_loss": 3.4772703279370116, + "tokens_seen": 1703870464 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492, + "loss": 1.5334, + "theoretical_loss": 3.477258822234026, + "tokens_seen": 1703936000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919797979797979, + "loss": 1.5056, + "theoretical_loss": 3.4772473170974614, + "tokens_seen": 1704001536 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491959595959596, + "loss": 1.1744, + "theoretical_loss": 3.477235812527267, + "tokens_seen": 1704067072 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491939393939394, + "loss": 1.5387, + "theoretical_loss": 3.4772243085233945, + "tokens_seen": 1704132608 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.3712732195854187, + "objective/train/avg_token_score": 0.018682070076465607, + "objective/train/docs_used": 965825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2413780689239502, + "objective/train/original_loss": 4.305376052856445, + "objective/train/theoretical_loss": 3.4772128050857933, + "objective/train/tokens_used": 63057376, + "objective/train/unlikelihood_loss": 0.054906729608774185, + "theoretical_loss": 3.4772128050857933, + "tokens_seen": 1704198144 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491919191919192, + "loss": 1.4768, + "theoretical_loss": 3.4772128050857933, + "tokens_seen": 1704198144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918989898989899, + "loss": 1.5636, + "theoretical_loss": 3.4772013022144144, + "tokens_seen": 1704263680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918787878787879, + "loss": 1.3053, + "theoretical_loss": 3.4771897999092083, + "tokens_seen": 1704329216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918585858585859, + "loss": 1.5969, + "theoretical_loss": 3.4771782981701245, + "tokens_seen": 1704394752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918383838383839, + "loss": 1.5039, + "theoretical_loss": 3.4771667969971136, + "tokens_seen": 1704460288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918181818181818, + "loss": 1.2291, + "theoretical_loss": 3.4771552963901264, + "tokens_seen": 1704525824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917979797979798, + "loss": 1.3308, + "theoretical_loss": 3.477143796349113, + "tokens_seen": 1704591360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917777777777777, + "loss": 1.4503, + "theoretical_loss": 3.477132296874024, + "tokens_seen": 1704656896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917575757575758, + "loss": 1.4605, + "theoretical_loss": 3.47712079796481, + "tokens_seen": 1704722432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917373737373737, + "loss": 1.553, + "theoretical_loss": 3.4771092996214206, + "tokens_seen": 1704787968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917171717171718, + "loss": 1.5346, + "theoretical_loss": 3.4770978018438066, + "tokens_seen": 1704853504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916969696969697, + "loss": 1.5756, + "theoretical_loss": 3.4770863046319187, + "tokens_seen": 1704919040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916767676767677, + "loss": 1.4356, + "theoretical_loss": 3.4770748079857072, + "tokens_seen": 1704984576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916565656565657, + "loss": 1.441, + "theoretical_loss": 3.4770633119051224, + "tokens_seen": 1705050112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916363636363637, + "loss": 1.3771, + "theoretical_loss": 3.4770518163901145, + "tokens_seen": 1705115648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916161616161616, + "loss": 1.4827, + "theoretical_loss": 3.4770403214406347, + "tokens_seen": 1705181184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915959595959596, + "loss": 1.2737, + "theoretical_loss": 3.4770288270566327, + "tokens_seen": 1705246720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915757575757576, + "loss": 1.6622, + "theoretical_loss": 3.477017333238059, + "tokens_seen": 1705312256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915555555555556, + "loss": 1.3899, + "theoretical_loss": 3.4770058399848645, + "tokens_seen": 1705377792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915353535353535, + "loss": 1.4385, + "theoretical_loss": 3.476994347296999, + "tokens_seen": 1705443328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915151515151515, + "loss": 1.3491, + "theoretical_loss": 3.476982855174414, + "tokens_seen": 1705508864 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914949494949494, + "loss": 1.4771, + "theoretical_loss": 3.476971363617059, + "tokens_seen": 1705574400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914747474747475, + "loss": 1.3391, + "theoretical_loss": 3.476959872624885, + "tokens_seen": 1705639936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914545454545455, + "loss": 1.4106, + "theoretical_loss": 3.4769483821978424, + "tokens_seen": 1705705472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914343434343435, + "loss": 1.2723, + "theoretical_loss": 3.4769368923358814, + "tokens_seen": 1705771008 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.470796674489975, + "objective/train/avg_token_score": 0.008710848167538643, + "objective/train/docs_used": 966739, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.535874843597412, + "objective/train/original_loss": 3.7179017066955566, + "objective/train/theoretical_loss": 3.476925403038953, + "objective/train/tokens_used": 64695776, + "objective/train/unlikelihood_loss": 0.08725922554731369, + "theoretical_loss": 3.476925403038953, + "tokens_seen": 1705836544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914141414141414, + "loss": 1.7468, + "theoretical_loss": 3.476925403038953, + "tokens_seen": 1705836544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913939393939394, + "loss": 1.2985, + "theoretical_loss": 3.4769139143070076, + "tokens_seen": 1705902080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913737373737374, + "loss": 1.3906, + "theoretical_loss": 3.476902426139995, + "tokens_seen": 1705967616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913535353535354, + "loss": 1.5775, + "theoretical_loss": 3.4768909385378666, + "tokens_seen": 1706033152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913333333333333, + "loss": 1.3863, + "theoretical_loss": 3.4768794515005723, + "tokens_seen": 1706098688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913131313131313, + "loss": 1.3093, + "theoretical_loss": 3.476867965028063, + "tokens_seen": 1706164224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912929292929293, + "loss": 1.4804, + "theoretical_loss": 3.47685647912029, + "tokens_seen": 1706229760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912727272727272, + "loss": 1.4999, + "theoretical_loss": 3.476844993777202, + "tokens_seen": 1706295296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912525252525252, + "loss": 1.4509, + "theoretical_loss": 3.476833508998751, + "tokens_seen": 1706360832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912323232323233, + "loss": 1.3798, + "theoretical_loss": 3.4768220247848873, + "tokens_seen": 1706426368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912121212121213, + "loss": 1.6698, + "theoretical_loss": 3.4768105411355608, + "tokens_seen": 1706491904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911919191919192, + "loss": 1.5185, + "theoretical_loss": 3.476799058050723, + "tokens_seen": 1706557440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911717171717172, + "loss": 1.1568, + "theoretical_loss": 3.4767875755303237, + "tokens_seen": 1706622976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911515151515152, + "loss": 1.0498, + "theoretical_loss": 3.476776093574314, + "tokens_seen": 1706688512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911313131313131, + "loss": 1.3549, + "theoretical_loss": 3.4767646121826443, + "tokens_seen": 1706754048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911111111111111, + "loss": 1.4073, + "theoretical_loss": 3.4767531313552658, + "tokens_seen": 1706819584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910909090909091, + "loss": 1.3959, + "theoretical_loss": 3.4767416510921274, + "tokens_seen": 1706885120 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491070707070707, + "loss": 1.2292, + "theoretical_loss": 3.4767301713931813, + "tokens_seen": 1706950656 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491050505050505, + "loss": 1.4875, + "theoretical_loss": 3.4767186922583777, + "tokens_seen": 1707016192 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491030303030303, + "loss": 1.3406, + "theoretical_loss": 3.4767072136876673, + "tokens_seen": 1707081728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910101010101011, + "loss": 1.3963, + "theoretical_loss": 3.4766957356810004, + "tokens_seen": 1707147264 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490989898989899, + "loss": 1.3853, + "theoretical_loss": 3.4766842582383277, + "tokens_seen": 1707212800 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490969696969697, + "loss": 1.2996, + "theoretical_loss": 3.4766727813596, + "tokens_seen": 1707278336 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490949494949495, + "loss": 1.6414, + "theoretical_loss": 3.476661305044768, + "tokens_seen": 1707343872 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490929292929293, + "loss": 1.3811, + "theoretical_loss": 3.476649829293782, + "tokens_seen": 1707409408 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.4374389052391052, + "objective/train/avg_token_score": 0.005900775548070669, + "objective/train/docs_used": 967356, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5339224338531494, + "objective/train/original_loss": 4.014285087585449, + "objective/train/theoretical_loss": 3.4766383541065933, + "objective/train/tokens_used": 66334176, + "objective/train/unlikelihood_loss": 0.08221994340419769, + "theoretical_loss": 3.4766383541065933, + "tokens_seen": 1707474944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909090909090909, + "loss": 1.4133, + "theoretical_loss": 3.4766383541065933, + "tokens_seen": 1707474944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908888888888889, + "loss": 1.2069, + "theoretical_loss": 3.4766268794831516, + "tokens_seen": 1707540480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908686868686868, + "loss": 1.2704, + "theoretical_loss": 3.4766154054234084, + "tokens_seen": 1707606016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908484848484849, + "loss": 1.5368, + "theoretical_loss": 3.4766039319273143, + "tokens_seen": 1707671552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908282828282828, + "loss": 1.2068, + "theoretical_loss": 3.4765924589948196, + "tokens_seen": 1707737088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908080808080808, + "loss": 1.3801, + "theoretical_loss": 3.4765809866258754, + "tokens_seen": 1707802624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907878787878787, + "loss": 1.5021, + "theoretical_loss": 3.476569514820432, + "tokens_seen": 1707868160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907676767676768, + "loss": 1.599, + "theoretical_loss": 3.4765580435784402, + "tokens_seen": 1707933696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907474747474748, + "loss": 1.6612, + "theoretical_loss": 3.4765465728998506, + "tokens_seen": 1707999232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907272727272728, + "loss": 1.7327, + "theoretical_loss": 3.476535102784614, + "tokens_seen": 1708064768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907070707070707, + "loss": 1.3258, + "theoretical_loss": 3.4765236332326817, + "tokens_seen": 1708130304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906868686868687, + "loss": 1.2935, + "theoretical_loss": 3.4765121642440038, + "tokens_seen": 1708195840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906666666666667, + "loss": 1.477, + "theoretical_loss": 3.476500695818531, + "tokens_seen": 1708261376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906464646464647, + "loss": 1.5143, + "theoretical_loss": 3.476489227956214, + "tokens_seen": 1708326912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906262626262626, + "loss": 1.5016, + "theoretical_loss": 3.476477760657004, + "tokens_seen": 1708392448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906060606060606, + "loss": 1.3412, + "theoretical_loss": 3.4764662939208515, + "tokens_seen": 1708457984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905858585858585, + "loss": 1.3278, + "theoretical_loss": 3.476454827747707, + "tokens_seen": 1708523520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905656565656565, + "loss": 1.4474, + "theoretical_loss": 3.476443362137522, + "tokens_seen": 1708589056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905454545454546, + "loss": 1.3127, + "theoretical_loss": 3.4764318970902464, + "tokens_seen": 1708654592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905252525252526, + "loss": 1.5852, + "theoretical_loss": 3.4764204326058317, + "tokens_seen": 1708720128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905050505050505, + "loss": 1.328, + "theoretical_loss": 3.476408968684228, + "tokens_seen": 1708785664 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904848484848485, + "loss": 1.5521, + "theoretical_loss": 3.4763975053253864, + "tokens_seen": 1708851200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904646464646465, + "loss": 1.4926, + "theoretical_loss": 3.4763860425292576, + "tokens_seen": 1708916736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904444444444445, + "loss": 1.3039, + "theoretical_loss": 3.4763745802957926, + "tokens_seen": 1708982272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904242424242424, + "loss": 1.5436, + "theoretical_loss": 3.476363118624943, + "tokens_seen": 1709047808 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.4849095940589905, + "objective/train/avg_token_score": 0.0028560131322592497, + "objective/train/docs_used": 968332, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6418017148971558, + "objective/train/original_loss": 3.7804646492004395, + "objective/train/theoretical_loss": 3.4763516575166573, + "objective/train/tokens_used": 67972576, + "objective/train/unlikelihood_loss": 0.1123645082116127, + "theoretical_loss": 3.4763516575166573, + "tokens_seen": 1709113344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904040404040404, + "loss": 1.516, + "theoretical_loss": 3.4763516575166573, + "tokens_seen": 1709113344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903838383838384, + "loss": 1.3689, + "theoretical_loss": 3.476340196970889, + "tokens_seen": 1709178880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903636363636364, + "loss": 1.6257, + "theoretical_loss": 3.476328736987587, + "tokens_seen": 1709244416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903434343434343, + "loss": 1.3624, + "theoretical_loss": 3.476317277566703, + "tokens_seen": 1709309952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903232323232323, + "loss": 1.4273, + "theoretical_loss": 3.4763058187081874, + "tokens_seen": 1709375488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903030303030304, + "loss": 1.3265, + "theoretical_loss": 3.476294360411991, + "tokens_seen": 1709441024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902828282828283, + "loss": 1.6407, + "theoretical_loss": 3.4762829026780655, + "tokens_seen": 1709506560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902626262626263, + "loss": 1.5641, + "theoretical_loss": 3.4762714455063612, + "tokens_seen": 1709572096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902424242424243, + "loss": 1.4669, + "theoretical_loss": 3.4762599888968286, + "tokens_seen": 1709637632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902222222222222, + "loss": 1.4743, + "theoretical_loss": 3.476248532849419, + "tokens_seen": 1709703168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902020202020202, + "loss": 1.1262, + "theoretical_loss": 3.4762370773640834, + "tokens_seen": 1709768704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901818181818182, + "loss": 1.4312, + "theoretical_loss": 3.476225622440772, + "tokens_seen": 1709834240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901616161616162, + "loss": 1.3713, + "theoretical_loss": 3.4762141680794367, + "tokens_seen": 1709899776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901414141414141, + "loss": 1.4197, + "theoretical_loss": 3.476202714280028, + "tokens_seen": 1709965312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901212121212121, + "loss": 1.3616, + "theoretical_loss": 3.476191261042496, + "tokens_seen": 1710030848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901010101010101, + "loss": 1.3839, + "theoretical_loss": 3.4761798083667923, + "tokens_seen": 1710096384 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490080808080808, + "loss": 1.6839, + "theoretical_loss": 3.476168356252868, + "tokens_seen": 1710161920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900606060606061, + "loss": 1.3635, + "theoretical_loss": 3.476156904700674, + "tokens_seen": 1710227456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900404040404041, + "loss": 1.6225, + "theoretical_loss": 3.476145453710161, + "tokens_seen": 1710292992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900202020202021, + "loss": 1.3887, + "theoretical_loss": 3.4761340032812793, + "tokens_seen": 1710358528 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049, + "loss": 1.4664, + "theoretical_loss": 3.4761225534139806, + "tokens_seen": 1710424064 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489979797979798, + "loss": 1.388, + "theoretical_loss": 3.476111104108216, + "tokens_seen": 1710489600 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489959595959596, + "loss": 1.4894, + "theoretical_loss": 3.476099655363936, + "tokens_seen": 1710555136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899393939393939, + "loss": 1.2687, + "theoretical_loss": 3.476088207181092, + "tokens_seen": 1710620672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899191919191919, + "loss": 1.298, + "theoretical_loss": 3.4760767595596347, + "tokens_seen": 1710686208 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.5306084752082825, + "objective/train/avg_token_score": 0.01167221274226904, + "objective/train/docs_used": 968886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6835694313049316, + "objective/train/original_loss": 4.055310249328613, + "objective/train/theoretical_loss": 3.4760653124995144, + "objective/train/tokens_used": 69610976, + "objective/train/unlikelihood_loss": 0.05426410958170891, + "theoretical_loss": 3.4760653124995144, + "tokens_seen": 1710751744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898989898989899, + "loss": 1.7209, + "theoretical_loss": 3.4760653124995144, + "tokens_seen": 1710751744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898787878787878, + "loss": 1.2871, + "theoretical_loss": 3.4760538660006834, + "tokens_seen": 1710817280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898585858585858, + "loss": 1.4492, + "theoretical_loss": 3.4760424200630915, + "tokens_seen": 1710882816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898383838383839, + "loss": 1.4377, + "theoretical_loss": 3.4760309746866906, + "tokens_seen": 1710948352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898181818181819, + "loss": 1.4232, + "theoretical_loss": 3.476019529871431, + "tokens_seen": 1711013888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897979797979798, + "loss": 1.6333, + "theoretical_loss": 3.4760080856172637, + "tokens_seen": 1711079424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897777777777778, + "loss": 1.4597, + "theoretical_loss": 3.475996641924141, + "tokens_seen": 1711144960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897575757575758, + "loss": 1.5103, + "theoretical_loss": 3.475985198792012, + "tokens_seen": 1711210496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897373737373738, + "loss": 1.3766, + "theoretical_loss": 3.4759737562208284, + "tokens_seen": 1711276032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897171717171717, + "loss": 1.532, + "theoretical_loss": 3.4759623142105416, + "tokens_seen": 1711341568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896969696969697, + "loss": 1.4063, + "theoretical_loss": 3.4759508727611026, + "tokens_seen": 1711407104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896767676767676, + "loss": 1.3932, + "theoretical_loss": 3.475939431872462, + "tokens_seen": 1711472640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896565656565657, + "loss": 1.4003, + "theoretical_loss": 3.4759279915445713, + "tokens_seen": 1711538176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896363636363636, + "loss": 1.335, + "theoretical_loss": 3.4759165517773813, + "tokens_seen": 1711603712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896161616161617, + "loss": 1.1782, + "theoretical_loss": 3.4759051125708433, + "tokens_seen": 1711669248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895959595959596, + "loss": 1.2264, + "theoretical_loss": 3.475893673924908, + "tokens_seen": 1711734784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895757575757576, + "loss": 1.4453, + "theoretical_loss": 3.475882235839527, + "tokens_seen": 1711800320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895555555555556, + "loss": 1.4011, + "theoretical_loss": 3.4758707983146504, + "tokens_seen": 1711865856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895353535353536, + "loss": 1.4299, + "theoretical_loss": 3.47585936135023, + "tokens_seen": 1711931392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895151515151515, + "loss": 1.5037, + "theoretical_loss": 3.475847924946217, + "tokens_seen": 1711996928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894949494949495, + "loss": 1.5703, + "theoretical_loss": 3.475836489102562, + "tokens_seen": 1712062464 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894747474747475, + "loss": 1.4185, + "theoretical_loss": 3.4758250538192166, + "tokens_seen": 1712128000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894545454545455, + "loss": 1.6266, + "theoretical_loss": 3.4758136190961313, + "tokens_seen": 1712193536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894343434343434, + "loss": 1.6168, + "theoretical_loss": 3.475802184933258, + "tokens_seen": 1712259072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894141414141414, + "loss": 1.2173, + "theoretical_loss": 3.475790751330547, + "tokens_seen": 1712324608 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.4318181872367859, + "objective/train/avg_token_score": 0.056178782135248184, + "objective/train/docs_used": 970327, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3246653079986572, + "objective/train/original_loss": 3.9383597373962402, + "objective/train/theoretical_loss": 3.47577931828795, + "objective/train/tokens_used": 71249376, + "objective/train/unlikelihood_loss": 0.07031881809234619, + "theoretical_loss": 3.47577931828795, + "tokens_seen": 1712390144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893939393939393, + "loss": 1.4182, + "theoretical_loss": 3.47577931828795, + "tokens_seen": 1712390144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893737373737374, + "loss": 1.1773, + "theoretical_loss": 3.4757678858054177, + "tokens_seen": 1712455680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893535353535354, + "loss": 1.624, + "theoretical_loss": 3.4757564538829016, + "tokens_seen": 1712521216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893333333333334, + "loss": 1.4873, + "theoretical_loss": 3.4757450225203526, + "tokens_seen": 1712586752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893131313131313, + "loss": 1.3469, + "theoretical_loss": 3.475733591717722, + "tokens_seen": 1712652288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892929292929293, + "loss": 1.4407, + "theoretical_loss": 3.4757221614749607, + "tokens_seen": 1712717824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892727272727273, + "loss": 1.4212, + "theoretical_loss": 3.47571073179202, + "tokens_seen": 1712783360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892525252525253, + "loss": 1.3738, + "theoretical_loss": 3.4756993026688514, + "tokens_seen": 1712848896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892323232323232, + "loss": 1.3697, + "theoretical_loss": 3.475687874105406, + "tokens_seen": 1712914432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892121212121212, + "loss": 1.5092, + "theoretical_loss": 3.4756764461016347, + "tokens_seen": 1712979968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891919191919192, + "loss": 1.2167, + "theoretical_loss": 3.4756650186574882, + "tokens_seen": 1713045504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891717171717171, + "loss": 1.4599, + "theoretical_loss": 3.475653591772918, + "tokens_seen": 1713111040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891515151515151, + "loss": 1.61, + "theoretical_loss": 3.4756421654478764, + "tokens_seen": 1713176576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891313131313132, + "loss": 1.679, + "theoretical_loss": 3.475630739682313, + "tokens_seen": 1713242112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891111111111112, + "loss": 1.3736, + "theoretical_loss": 3.47561931447618, + "tokens_seen": 1713307648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890909090909091, + "loss": 1.2454, + "theoretical_loss": 3.475607889829428, + "tokens_seen": 1713373184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890707070707071, + "loss": 1.5084, + "theoretical_loss": 3.475596465742009, + "tokens_seen": 1713438720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890505050505051, + "loss": 1.5201, + "theoretical_loss": 3.475585042213874, + "tokens_seen": 1713504256 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489030303030303, + "loss": 1.6822, + "theoretical_loss": 3.4755736192449733, + "tokens_seen": 1713569792 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489010101010101, + "loss": 1.4501, + "theoretical_loss": 3.4755621968352592, + "tokens_seen": 1713635328 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488989898989899, + "loss": 1.4203, + "theoretical_loss": 3.475550774984682, + "tokens_seen": 1713700864 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889696969696969, + "loss": 1.4855, + "theoretical_loss": 3.4755393536931942, + "tokens_seen": 1713766400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889494949494949, + "loss": 1.3609, + "theoretical_loss": 3.4755279329607465, + "tokens_seen": 1713831936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889292929292929, + "loss": 1.5888, + "theoretical_loss": 3.4755165127872893, + "tokens_seen": 1713897472 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488909090909091, + "loss": 1.5043, + "theoretical_loss": 3.4755050931727753, + "tokens_seen": 1713963008 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.5464931726455688, + "objective/train/avg_token_score": 0.0072470419108867645, + "objective/train/docs_used": 971016, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7423732280731201, + "objective/train/original_loss": 3.5915262699127197, + "objective/train/theoretical_loss": 3.4754936741171543, + "objective/train/tokens_used": 72887776, + "objective/train/unlikelihood_loss": 0.07072310149669647, + "theoretical_loss": 3.4754936741171543, + "tokens_seen": 1714028544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888888888888889, + "loss": 1.4365, + "theoretical_loss": 3.4754936741171543, + "tokens_seen": 1714028544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888686868686869, + "loss": 1.434, + "theoretical_loss": 3.475482255620379, + "tokens_seen": 1714094080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888484848484849, + "loss": 1.1613, + "theoretical_loss": 3.4754708376823995, + "tokens_seen": 1714159616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888282828282829, + "loss": 1.2823, + "theoretical_loss": 3.475459420303168, + "tokens_seen": 1714225152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888080808080808, + "loss": 1.2084, + "theoretical_loss": 3.4754480034826356, + "tokens_seen": 1714290688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887878787878788, + "loss": 1.1509, + "theoretical_loss": 3.475436587220753, + "tokens_seen": 1714356224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887676767676768, + "loss": 1.4293, + "theoretical_loss": 3.4754251715174727, + "tokens_seen": 1714421760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887474747474748, + "loss": 1.4948, + "theoretical_loss": 3.4754137563727445, + "tokens_seen": 1714487296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887272727272727, + "loss": 1.3427, + "theoretical_loss": 3.4754023417865207, + "tokens_seen": 1714552832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887070707070707, + "loss": 1.393, + "theoretical_loss": 3.4753909277587525, + "tokens_seen": 1714618368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886868686868686, + "loss": 1.4489, + "theoretical_loss": 3.475379514289391, + "tokens_seen": 1714683904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886666666666667, + "loss": 1.3675, + "theoretical_loss": 3.475368101378388, + "tokens_seen": 1714749440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886464646464647, + "loss": 1.2169, + "theoretical_loss": 3.475356689025694, + "tokens_seen": 1714814976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886262626262627, + "loss": 1.5594, + "theoretical_loss": 3.4753452772312614, + "tokens_seen": 1714880512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886060606060606, + "loss": 1.3374, + "theoretical_loss": 3.475333865995041, + "tokens_seen": 1714946048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885858585858586, + "loss": 1.3952, + "theoretical_loss": 3.4753224553169835, + "tokens_seen": 1715011584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885656565656566, + "loss": 1.4714, + "theoretical_loss": 3.4753110451970417, + "tokens_seen": 1715077120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885454545454546, + "loss": 1.2954, + "theoretical_loss": 3.475299635635166, + "tokens_seen": 1715142656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885252525252525, + "loss": 1.5126, + "theoretical_loss": 3.475288226631308, + "tokens_seen": 1715208192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885050505050505, + "loss": 1.307, + "theoretical_loss": 3.475276818185419, + "tokens_seen": 1715273728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884848484848484, + "loss": 1.3257, + "theoretical_loss": 3.475265410297451, + "tokens_seen": 1715339264 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884646464646465, + "loss": 1.3537, + "theoretical_loss": 3.4752540029673544, + "tokens_seen": 1715404800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884444444444445, + "loss": 1.3866, + "theoretical_loss": 3.4752425961950815, + "tokens_seen": 1715470336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884242424242425, + "loss": 1.4019, + "theoretical_loss": 3.4752311899805832, + "tokens_seen": 1715535872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884040404040404, + "loss": 1.1464, + "theoretical_loss": 3.475219784323811, + "tokens_seen": 1715601408 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.4002932608127594, + "objective/train/avg_token_score": 0.017858585342764854, + "objective/train/docs_used": 972281, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3489974737167358, + "objective/train/original_loss": 4.905088424682617, + "objective/train/theoretical_loss": 3.475208379224716, + "objective/train/tokens_used": 74526176, + "objective/train/unlikelihood_loss": 0.05417528748512268, + "theoretical_loss": 3.475208379224716, + "tokens_seen": 1715666944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883838383838384, + "loss": 1.526, + "theoretical_loss": 3.475208379224716, + "tokens_seen": 1715666944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883636363636364, + "loss": 1.2685, + "theoretical_loss": 3.475196974683251, + "tokens_seen": 1715732480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883434343434344, + "loss": 1.4254, + "theoretical_loss": 3.4751855706993657, + "tokens_seen": 1715798016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883232323232323, + "loss": 1.3064, + "theoretical_loss": 3.4751741672730123, + "tokens_seen": 1715863552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883030303030303, + "loss": 1.3617, + "theoretical_loss": 3.4751627644041427, + "tokens_seen": 1715929088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882828282828283, + "loss": 1.4178, + "theoretical_loss": 3.4751513620927073, + "tokens_seen": 1715994624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882626262626263, + "loss": 1.6226, + "theoretical_loss": 3.475139960338659, + "tokens_seen": 1716060160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882424242424243, + "loss": 1.4401, + "theoretical_loss": 3.4751285591419476, + "tokens_seen": 1716125696 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048822222222222225, + "loss": 1.3443, + "theoretical_loss": 3.475117158502526, + "tokens_seen": 1716191232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882020202020202, + "loss": 1.4155, + "theoretical_loss": 3.4751057584203444, + "tokens_seen": 1716256768 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048818181818181816, + "loss": 1.2397, + "theoretical_loss": 3.4750943588953556, + "tokens_seen": 1716322304 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048816161616161617, + "loss": 1.2896, + "theoretical_loss": 3.47508295992751, + "tokens_seen": 1716387840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881414141414142, + "loss": 1.4234, + "theoretical_loss": 3.47507156151676, + "tokens_seen": 1716453376 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048812121212121214, + "loss": 1.6226, + "theoretical_loss": 3.4750601636630565, + "tokens_seen": 1716518912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881010101010101, + "loss": 1.3232, + "theoretical_loss": 3.475048766366351, + "tokens_seen": 1716584448 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048808080808080805, + "loss": 1.2095, + "theoretical_loss": 3.4750373696265955, + "tokens_seen": 1716649984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880606060606061, + "loss": 1.3368, + "theoretical_loss": 3.4750259734437408, + "tokens_seen": 1716715520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880404040404041, + "loss": 1.5763, + "theoretical_loss": 3.4750145778177393, + "tokens_seen": 1716781056 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048802020202020203, + "loss": 1.537, + "theoretical_loss": 3.4750031827485417, + "tokens_seen": 1716846592 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488, + "loss": 1.4221, + "theoretical_loss": 3.4749917882360997, + "tokens_seen": 1716912128 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048797979797979795, + "loss": 1.4776, + "theoretical_loss": 3.4749803942803656, + "tokens_seen": 1716977664 + }, + { + "epoch": 0.03, + "learning_rate": 0.000487959595959596, + "loss": 1.6911, + "theoretical_loss": 3.4749690008812903, + "tokens_seen": 1717043200 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048793939393939397, + "loss": 1.6163, + "theoretical_loss": 3.4749576080388254, + "tokens_seen": 1717108736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879191919191919, + "loss": 1.5933, + "theoretical_loss": 3.4749462157529227, + "tokens_seen": 1717174272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878989898989899, + "loss": 1.4458, + "theoretical_loss": 3.4749348240235336, + "tokens_seen": 1717239808 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.34756842255592346, + "objective/train/avg_token_score": 0.022121969610452652, + "objective/train/docs_used": 972817, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2492351531982422, + "objective/train/original_loss": 4.464137554168701, + "objective/train/theoretical_loss": 3.47492343285061, + "objective/train/tokens_used": 76164576, + "objective/train/unlikelihood_loss": 0.06865523010492325, + "theoretical_loss": 3.47492343285061, + "tokens_seen": 1717305344 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048787878787878784, + "loss": 1.4663, + "theoretical_loss": 3.47492343285061, + "tokens_seen": 1717305344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878585858585859, + "loss": 1.6475, + "theoretical_loss": 3.474912042234102, + "tokens_seen": 1717370880 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048783838383838386, + "loss": 1.3615, + "theoretical_loss": 3.4749006521739636, + "tokens_seen": 1717436416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878181818181818, + "loss": 1.3724, + "theoretical_loss": 3.4748892626701444, + "tokens_seen": 1717501952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877979797979798, + "loss": 1.4726, + "theoretical_loss": 3.4748778737225976, + "tokens_seen": 1717567488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877777777777778, + "loss": 1.5526, + "theoretical_loss": 3.4748664853312734, + "tokens_seen": 1717633024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877575757575758, + "loss": 1.3223, + "theoretical_loss": 3.474855097496124, + "tokens_seen": 1717698560 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048773737373737375, + "loss": 1.6783, + "theoretical_loss": 3.474843710217101, + "tokens_seen": 1717764096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877171717171717, + "loss": 1.3255, + "theoretical_loss": 3.474832323494156, + "tokens_seen": 1717829632 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048769696969696967, + "loss": 1.3216, + "theoretical_loss": 3.474820937327241, + "tokens_seen": 1717895168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876767676767677, + "loss": 1.3008, + "theoretical_loss": 3.474809551716307, + "tokens_seen": 1717960704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876565656565657, + "loss": 1.1347, + "theoretical_loss": 3.474798166661306, + "tokens_seen": 1718026240 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048763636363636365, + "loss": 1.4412, + "theoretical_loss": 3.4747867821621896, + "tokens_seen": 1718091776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876161616161616, + "loss": 1.1156, + "theoretical_loss": 3.4747753982189096, + "tokens_seen": 1718157312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875959595959596, + "loss": 1.4876, + "theoretical_loss": 3.474764014831417, + "tokens_seen": 1718222848 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048757575757575757, + "loss": 1.4029, + "theoretical_loss": 3.4747526319996647, + "tokens_seen": 1718288384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875555555555556, + "loss": 1.3172, + "theoretical_loss": 3.474741249723603, + "tokens_seen": 1718353920 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048753535353535354, + "loss": 1.2698, + "theoretical_loss": 3.4747298680031844, + "tokens_seen": 1718419456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875151515151515, + "loss": 1.5606, + "theoretical_loss": 3.4747184868383605, + "tokens_seen": 1718484992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874949494949495, + "loss": 1.4975, + "theoretical_loss": 3.474707106229083, + "tokens_seen": 1718550528 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048747474747474746, + "loss": 1.5241, + "theoretical_loss": 3.4746957261753035, + "tokens_seen": 1718616064 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874545454545455, + "loss": 1.5173, + "theoretical_loss": 3.4746843466769737, + "tokens_seen": 1718681600 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048743434343434343, + "loss": 1.5222, + "theoretical_loss": 3.4746729677340453, + "tokens_seen": 1718747136 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048741414141414144, + "loss": 1.4428, + "theoretical_loss": 3.47466158934647, + "tokens_seen": 1718812672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004873939393939394, + "loss": 1.2955, + "theoretical_loss": 3.4746502115141995, + "tokens_seen": 1718878208 + }, + { + "epoch": 0.03, + "objective/train/aligned_token_count": 0.3179985284805298, + "objective/train/avg_token_score": 0.02353089302778244, + "objective/train/docs_used": 973424, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0046734809875488, + "objective/train/original_loss": 4.215363502502441, + "objective/train/theoretical_loss": 3.4746388342371857, + "objective/train/tokens_used": 77802976, + "objective/train/unlikelihood_loss": 0.07478584349155426, + "theoretical_loss": 3.4746388342371857, + "tokens_seen": 1718943744 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048737373737373736, + "loss": 1.2882, + "theoretical_loss": 3.4746388342371857, + "tokens_seen": 1718943744 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048735353535353537, + "loss": 1.5482, + "theoretical_loss": 3.47462745751538, + "tokens_seen": 1719009280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873333333333333, + "loss": 1.1838, + "theoretical_loss": 3.474616081348735, + "tokens_seen": 1719074816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048731313131313134, + "loss": 1.438, + "theoretical_loss": 3.4746047057372014, + "tokens_seen": 1719140352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872929292929293, + "loss": 1.4355, + "theoretical_loss": 3.4745933306807313, + "tokens_seen": 1719205888 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048727272727272725, + "loss": 1.2308, + "theoretical_loss": 3.474581956179277, + "tokens_seen": 1719271424 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048725252525252526, + "loss": 1.5441, + "theoretical_loss": 3.474570582232789, + "tokens_seen": 1719336960 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048723232323232327, + "loss": 1.4567, + "theoretical_loss": 3.4745592088412205, + "tokens_seen": 1719402496 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048721212121212123, + "loss": 1.5781, + "theoretical_loss": 3.4745478360045228, + "tokens_seen": 1719468032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871919191919192, + "loss": 1.4201, + "theoretical_loss": 3.4745364637226466, + "tokens_seen": 1719533568 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048717171717171714, + "loss": 1.5531, + "theoretical_loss": 3.4745250919955453, + "tokens_seen": 1719599104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048715151515151515, + "loss": 1.0908, + "theoretical_loss": 3.47451372082317, + "tokens_seen": 1719664640 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048713131313131316, + "loss": 1.6216, + "theoretical_loss": 3.474502350205472, + "tokens_seen": 1719730176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871111111111111, + "loss": 1.2598, + "theoretical_loss": 3.474490980142404, + "tokens_seen": 1719795712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870909090909091, + "loss": 1.3527, + "theoretical_loss": 3.4744796106339173, + "tokens_seen": 1719861248 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048707070707070704, + "loss": 1.3626, + "theoretical_loss": 3.474468241679964, + "tokens_seen": 1719926784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870505050505051, + "loss": 1.4276, + "theoretical_loss": 3.474456873280496, + "tokens_seen": 1719992320 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048703030303030306, + "loss": 1.5703, + "theoretical_loss": 3.4744455054354644, + "tokens_seen": 1720057856 + }, + { + "epoch": 0.04, + "learning_rate": 0.000487010101010101, + "loss": 1.4335, + "theoretical_loss": 3.474434138144822, + "tokens_seen": 1720123392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048698989898989897, + "loss": 1.4018, + "theoretical_loss": 3.47442277140852, + "tokens_seen": 1720188928 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486969696969697, + "loss": 1.4003, + "theoretical_loss": 3.47441140522651, + "tokens_seen": 1720254464 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486949494949495, + "loss": 1.5975, + "theoretical_loss": 3.474400039598745, + "tokens_seen": 1720320000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048692929292929295, + "loss": 1.3209, + "theoretical_loss": 3.4743886745251755, + "tokens_seen": 1720385536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869090909090909, + "loss": 1.5469, + "theoretical_loss": 3.4743773100057544, + "tokens_seen": 1720451072 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048688888888888886, + "loss": 1.3673, + "theoretical_loss": 3.4743659460404333, + "tokens_seen": 1720516608 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.5211998820304871, + "objective/train/avg_token_score": 0.0027845173608511686, + "objective/train/docs_used": 974650, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7277867794036865, + "objective/train/original_loss": 3.4647514820098877, + "objective/train/theoretical_loss": 3.4743545826291635, + "objective/train/tokens_used": 79441376, + "objective/train/unlikelihood_loss": 0.10759128630161285, + "theoretical_loss": 3.4743545826291635, + "tokens_seen": 1720582144 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048686868686868693, + "loss": 1.5171, + "theoretical_loss": 3.4743545826291635, + "tokens_seen": 1720582144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868484848484849, + "loss": 1.3346, + "theoretical_loss": 3.4743432197718978, + "tokens_seen": 1720647680 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048682828282828284, + "loss": 1.2947, + "theoretical_loss": 3.4743318574685875, + "tokens_seen": 1720713216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868080808080808, + "loss": 1.3649, + "theoretical_loss": 3.4743204957191844, + "tokens_seen": 1720778752 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048678787878787876, + "loss": 1.3602, + "theoretical_loss": 3.474309134523641, + "tokens_seen": 1720844288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867676767676768, + "loss": 1.3379, + "theoretical_loss": 3.4742977738819087, + "tokens_seen": 1720909824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867474747474748, + "loss": 1.5094, + "theoretical_loss": 3.4742864137939398, + "tokens_seen": 1720975360 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048672727272727274, + "loss": 1.4478, + "theoretical_loss": 3.474275054259685, + "tokens_seen": 1721040896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867070707070707, + "loss": 1.4563, + "theoretical_loss": 3.4742636952790984, + "tokens_seen": 1721106432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866868686868687, + "loss": 1.5572, + "theoretical_loss": 3.47425233685213, + "tokens_seen": 1721171968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866666666666667, + "loss": 1.3569, + "theoretical_loss": 3.474240978978733, + "tokens_seen": 1721237504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048664646464646467, + "loss": 1.2837, + "theoretical_loss": 3.474229621658859, + "tokens_seen": 1721303040 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048662626262626263, + "loss": 1.4573, + "theoretical_loss": 3.474218264892459, + "tokens_seen": 1721368576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866060606060606, + "loss": 1.4406, + "theoretical_loss": 3.4742069086794864, + "tokens_seen": 1721434112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865858585858586, + "loss": 1.2444, + "theoretical_loss": 3.474195553019892, + "tokens_seen": 1721499648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865656565656566, + "loss": 1.3948, + "theoretical_loss": 3.4741841979136288, + "tokens_seen": 1721565184 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048654545454545456, + "loss": 1.4, + "theoretical_loss": 3.4741728433606482, + "tokens_seen": 1721630720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865252525252525, + "loss": 1.5166, + "theoretical_loss": 3.4741614893609016, + "tokens_seen": 1721696256 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048650505050505053, + "loss": 1.5006, + "theoretical_loss": 3.4741501359143423, + "tokens_seen": 1721761792 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864848484848485, + "loss": 1.2737, + "theoretical_loss": 3.4741387830209214, + "tokens_seen": 1721827328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864646464646465, + "loss": 1.3364, + "theoretical_loss": 3.474127430680591, + "tokens_seen": 1721892864 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048644444444444446, + "loss": 1.4177, + "theoretical_loss": 3.474116078893303, + "tokens_seen": 1721958400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864242424242424, + "loss": 1.3975, + "theoretical_loss": 3.47410472765901, + "tokens_seen": 1722023936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864040404040404, + "loss": 1.3059, + "theoretical_loss": 3.474093376977664, + "tokens_seen": 1722089472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863838383838384, + "loss": 1.4123, + "theoretical_loss": 3.474082026849216, + "tokens_seen": 1722155008 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.3375488817691803, + "objective/train/avg_token_score": 0.013777061365544796, + "objective/train/docs_used": 975130, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2517693042755127, + "objective/train/original_loss": 5.1249237060546875, + "objective/train/theoretical_loss": 3.4740706772736187, + "objective/train/tokens_used": 81079776, + "objective/train/unlikelihood_loss": 0.040009669959545135, + "theoretical_loss": 3.4740706772736187, + "tokens_seen": 1722220544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863636363636364, + "loss": 1.5127, + "theoretical_loss": 3.4740706772736187, + "tokens_seen": 1722220544 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048634343434343435, + "loss": 1.2964, + "theoretical_loss": 3.4740593282508243, + "tokens_seen": 1722286080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048632323232323236, + "loss": 1.4136, + "theoretical_loss": 3.474047979780785, + "tokens_seen": 1722351616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863030303030303, + "loss": 1.4117, + "theoretical_loss": 3.474036631863452, + "tokens_seen": 1722417152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862828282828283, + "loss": 1.3667, + "theoretical_loss": 3.4740252844987785, + "tokens_seen": 1722482688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862626262626263, + "loss": 1.2363, + "theoretical_loss": 3.474013937686715, + "tokens_seen": 1722548224 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048624242424242424, + "loss": 1.4347, + "theoretical_loss": 3.4740025914272152, + "tokens_seen": 1722613760 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048622222222222225, + "loss": 1.3785, + "theoretical_loss": 3.4739912457202307, + "tokens_seen": 1722679296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862020202020202, + "loss": 1.4418, + "theoretical_loss": 3.4739799005657126, + "tokens_seen": 1722744832 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048618181818181817, + "loss": 1.4983, + "theoretical_loss": 3.473968555963614, + "tokens_seen": 1722810368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861616161616162, + "loss": 1.4721, + "theoretical_loss": 3.473957211913887, + "tokens_seen": 1722875904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861414141414142, + "loss": 1.4795, + "theoretical_loss": 3.4739458684164832, + "tokens_seen": 1722941440 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048612121212121215, + "loss": 1.3607, + "theoretical_loss": 3.473934525471355, + "tokens_seen": 1723006976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861010101010101, + "loss": 1.4223, + "theoretical_loss": 3.4739231830784543, + "tokens_seen": 1723072512 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048608080808080806, + "loss": 1.5619, + "theoretical_loss": 3.473911841237733, + "tokens_seen": 1723138048 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048606060606060607, + "loss": 1.4897, + "theoretical_loss": 3.473900499949144, + "tokens_seen": 1723203584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860404040404041, + "loss": 1.3993, + "theoretical_loss": 3.4738891592126393, + "tokens_seen": 1723269120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048602020202020204, + "loss": 1.408, + "theoretical_loss": 3.4738778190281705, + "tokens_seen": 1723334656 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486, + "loss": 1.3879, + "theoretical_loss": 3.4738664793956895, + "tokens_seen": 1723400192 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048597979797979795, + "loss": 1.4359, + "theoretical_loss": 3.473855140315149, + "tokens_seen": 1723465728 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048595959595959596, + "loss": 1.4494, + "theoretical_loss": 3.473843801786501, + "tokens_seen": 1723531264 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485939393939394, + "loss": 1.3349, + "theoretical_loss": 3.473832463809698, + "tokens_seen": 1723596800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048591919191919193, + "loss": 1.4685, + "theoretical_loss": 3.4738211263846916, + "tokens_seen": 1723662336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858989898989899, + "loss": 1.3586, + "theoretical_loss": 3.473809789511434, + "tokens_seen": 1723727872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048587878787878784, + "loss": 1.3079, + "theoretical_loss": 3.473798453189878, + "tokens_seen": 1723793408 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.4409824013710022, + "objective/train/avg_token_score": 0.026003144681453705, + "objective/train/docs_used": 976352, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4241589307785034, + "objective/train/original_loss": 4.103183746337891, + "objective/train/theoretical_loss": 3.4737871174199757, + "objective/train/tokens_used": 82718176, + "objective/train/unlikelihood_loss": 0.09437726438045502, + "theoretical_loss": 3.4737871174199757, + "tokens_seen": 1723858944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858585858585859, + "loss": 1.4386, + "theoretical_loss": 3.4737871174199757, + "tokens_seen": 1723858944 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048583838383838387, + "loss": 1.3626, + "theoretical_loss": 3.4737757822016784, + "tokens_seen": 1723924480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858181818181818, + "loss": 1.4696, + "theoretical_loss": 3.473764447534939, + "tokens_seen": 1723990016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857979797979798, + "loss": 1.3754, + "theoretical_loss": 3.4737531134197095, + "tokens_seen": 1724055552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048577777777777774, + "loss": 1.4346, + "theoretical_loss": 3.473741779855942, + "tokens_seen": 1724121088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857575757575758, + "loss": 1.4929, + "theoretical_loss": 3.4737304468435894, + "tokens_seen": 1724186624 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048573737373737376, + "loss": 1.4366, + "theoretical_loss": 3.473719114382603, + "tokens_seen": 1724252160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857171717171717, + "loss": 1.2766, + "theoretical_loss": 3.4737077824729354, + "tokens_seen": 1724317696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856969696969697, + "loss": 1.3714, + "theoretical_loss": 3.4736964511145385, + "tokens_seen": 1724383232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856767676767677, + "loss": 1.2763, + "theoretical_loss": 3.473685120307365, + "tokens_seen": 1724448768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856565656565657, + "loss": 1.4836, + "theoretical_loss": 3.4736737900513672, + "tokens_seen": 1724514304 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048563636363636365, + "loss": 1.4698, + "theoretical_loss": 3.473662460346497, + "tokens_seen": 1724579840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856161616161616, + "loss": 1.4449, + "theoretical_loss": 3.4736511311927067, + "tokens_seen": 1724645376 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048559595959595957, + "loss": 1.3052, + "theoretical_loss": 3.4736398025899486, + "tokens_seen": 1724710912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855757575757576, + "loss": 1.565, + "theoretical_loss": 3.4736284745381756, + "tokens_seen": 1724776448 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855555555555556, + "loss": 1.3919, + "theoretical_loss": 3.4736171470373387, + "tokens_seen": 1724841984 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048553535353535355, + "loss": 1.5023, + "theoretical_loss": 3.473605820087391, + "tokens_seen": 1724907520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855151515151515, + "loss": 1.2998, + "theoretical_loss": 3.4735944936882848, + "tokens_seen": 1724973056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854949494949495, + "loss": 1.6207, + "theoretical_loss": 3.473583167839972, + "tokens_seen": 1725038592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048547474747474747, + "loss": 1.2696, + "theoretical_loss": 3.473571842542405, + "tokens_seen": 1725104128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854545454545455, + "loss": 1.6765, + "theoretical_loss": 3.473560517795536, + "tokens_seen": 1725169664 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048543434343434344, + "loss": 1.5439, + "theoretical_loss": 3.473549193599318, + "tokens_seen": 1725235200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854141414141414, + "loss": 1.5656, + "theoretical_loss": 3.4735378699537027, + "tokens_seen": 1725300736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853939393939394, + "loss": 1.2932, + "theoretical_loss": 3.4735265468586425, + "tokens_seen": 1725366272 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048537373737373736, + "loss": 1.4775, + "theoretical_loss": 3.4735152243140894, + "tokens_seen": 1725431808 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.4415322542190552, + "objective/train/avg_token_score": 0.00970387365669012, + "objective/train/docs_used": 977030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4506078958511353, + "objective/train/original_loss": 4.433774948120117, + "objective/train/theoretical_loss": 3.4735039023199965, + "objective/train/tokens_used": 84356576, + "objective/train/unlikelihood_loss": 0.02620289847254753, + "theoretical_loss": 3.4735039023199965, + "tokens_seen": 1725497344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853535353535354, + "loss": 1.7168, + "theoretical_loss": 3.4735039023199965, + "tokens_seen": 1725497344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048533333333333333, + "loss": 1.2021, + "theoretical_loss": 3.4734925808763153, + "tokens_seen": 1725562880 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048531313131313134, + "loss": 1.4966, + "theoretical_loss": 3.473481259982999, + "tokens_seen": 1725628416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852929292929293, + "loss": 1.4124, + "theoretical_loss": 3.4734699396399993, + "tokens_seen": 1725693952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852727272727273, + "loss": 1.3887, + "theoretical_loss": 3.4734586198472686, + "tokens_seen": 1725759488 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048525252525252527, + "loss": 1.1216, + "theoretical_loss": 3.4734473006047595, + "tokens_seen": 1725825024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852323232323232, + "loss": 1.3029, + "theoretical_loss": 3.4734359819124245, + "tokens_seen": 1725890560 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048521212121212123, + "loss": 1.3902, + "theoretical_loss": 3.4734246637702153, + "tokens_seen": 1725956096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851919191919192, + "loss": 1.34, + "theoretical_loss": 3.4734133461780847, + "tokens_seen": 1726021632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851717171717172, + "loss": 1.4358, + "theoretical_loss": 3.4734020291359853, + "tokens_seen": 1726087168 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048515151515151516, + "loss": 1.4642, + "theoretical_loss": 3.473390712643869, + "tokens_seen": 1726152704 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048513131313131317, + "loss": 1.3417, + "theoretical_loss": 3.473379396701689, + "tokens_seen": 1726218240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851111111111111, + "loss": 1.1703, + "theoretical_loss": 3.473368081309397, + "tokens_seen": 1726283776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850909090909091, + "loss": 1.6721, + "theoretical_loss": 3.473356766466945, + "tokens_seen": 1726349312 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850707070707071, + "loss": 1.3141, + "theoretical_loss": 3.4733454521742866, + "tokens_seen": 1726414848 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048505050505050505, + "loss": 1.2502, + "theoretical_loss": 3.4733341384313734, + "tokens_seen": 1726480384 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048503030303030306, + "loss": 1.2986, + "theoretical_loss": 3.4733228252381574, + "tokens_seen": 1726545920 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485010101010101, + "loss": 1.3311, + "theoretical_loss": 3.4733115125945924, + "tokens_seen": 1726611456 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484989898989899, + "loss": 1.5198, + "theoretical_loss": 3.47330020050063, + "tokens_seen": 1726676992 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484969696969697, + "loss": 1.2449, + "theoretical_loss": 3.4732888889562226, + "tokens_seen": 1726742528 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484949494949495, + "loss": 1.345, + "theoretical_loss": 3.4732775779613227, + "tokens_seen": 1726808064 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048492929292929296, + "loss": 1.3956, + "theoretical_loss": 3.473266267515883, + "tokens_seen": 1726873600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849090909090909, + "loss": 1.3801, + "theoretical_loss": 3.4732549576198553, + "tokens_seen": 1726939136 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048488888888888887, + "loss": 1.3834, + "theoretical_loss": 3.4732436482731925, + "tokens_seen": 1727004672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848686868686869, + "loss": 1.4937, + "theoretical_loss": 3.4732323394758478, + "tokens_seen": 1727070208 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.39345064759254456, + "objective/train/avg_token_score": 0.009139729663729668, + "objective/train/docs_used": 978106, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4235575199127197, + "objective/train/original_loss": 4.360210418701172, + "objective/train/theoretical_loss": 3.473221031227773, + "objective/train/tokens_used": 85994976, + "objective/train/unlikelihood_loss": 0.08843415230512619, + "theoretical_loss": 3.473221031227773, + "tokens_seen": 1727135744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848484848484849, + "loss": 1.5847, + "theoretical_loss": 3.473221031227773, + "tokens_seen": 1727135744 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048482828282828285, + "loss": 1.4794, + "theoretical_loss": 3.47320972352892, + "tokens_seen": 1727201280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848080808080808, + "loss": 1.4167, + "theoretical_loss": 3.473198416379242, + "tokens_seen": 1727266816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048478787878787876, + "loss": 1.2351, + "theoretical_loss": 3.4731871097786913, + "tokens_seen": 1727332352 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048476767676767683, + "loss": 1.3563, + "theoretical_loss": 3.4731758037272202, + "tokens_seen": 1727397888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847474747474748, + "loss": 1.291, + "theoretical_loss": 3.473164498224782, + "tokens_seen": 1727463424 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048472727272727274, + "loss": 1.4047, + "theoretical_loss": 3.4731531932713287, + "tokens_seen": 1727528960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847070707070707, + "loss": 1.3456, + "theoretical_loss": 3.473141888866812, + "tokens_seen": 1727594496 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048468686868686865, + "loss": 1.3892, + "theoretical_loss": 3.4731305850111855, + "tokens_seen": 1727660032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846666666666667, + "loss": 1.3859, + "theoretical_loss": 3.473119281704402, + "tokens_seen": 1727725568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846464646464647, + "loss": 1.1975, + "theoretical_loss": 3.473107978946413, + "tokens_seen": 1727791104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048462626262626263, + "loss": 1.3792, + "theoretical_loss": 3.4730966767371716, + "tokens_seen": 1727856640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846060606060606, + "loss": 1.3253, + "theoretical_loss": 3.4730853750766304, + "tokens_seen": 1727922176 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048458585858585855, + "loss": 1.5187, + "theoretical_loss": 3.4730740739647414, + "tokens_seen": 1727987712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845656565656566, + "loss": 1.4738, + "theoretical_loss": 3.4730627734014576, + "tokens_seen": 1728053248 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048454545454545457, + "loss": 1.1208, + "theoretical_loss": 3.4730514733867315, + "tokens_seen": 1728118784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845252525252525, + "loss": 1.3628, + "theoretical_loss": 3.4730401739205163, + "tokens_seen": 1728184320 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845050505050505, + "loss": 1.4873, + "theoretical_loss": 3.4730288750027634, + "tokens_seen": 1728249856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844848484848485, + "loss": 1.3379, + "theoretical_loss": 3.473017576633426, + "tokens_seen": 1728315392 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844646464646465, + "loss": 1.2096, + "theoretical_loss": 3.4730062788124565, + "tokens_seen": 1728380928 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048444444444444446, + "loss": 1.2331, + "theoretical_loss": 3.472994981539808, + "tokens_seen": 1728446464 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844242424242424, + "loss": 1.5172, + "theoretical_loss": 3.4729836848154325, + "tokens_seen": 1728512000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844040404040404, + "loss": 1.4394, + "theoretical_loss": 3.472972388639283, + "tokens_seen": 1728577536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843838383838384, + "loss": 1.4188, + "theoretical_loss": 3.4729610930113117, + "tokens_seen": 1728643072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843636363636364, + "loss": 1.305, + "theoretical_loss": 3.472949797931472, + "tokens_seen": 1728708608 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.3660801649093628, + "objective/train/avg_token_score": 0.007394853513687849, + "objective/train/docs_used": 978835, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.256290316581726, + "objective/train/original_loss": 4.4458208084106445, + "objective/train/theoretical_loss": 3.472938503399715, + "objective/train/tokens_used": 87633376, + "objective/train/unlikelihood_loss": 0.06266473978757858, + "theoretical_loss": 3.472938503399715, + "tokens_seen": 1728774144 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048434343434343435, + "loss": 1.5173, + "theoretical_loss": 3.472938503399715, + "tokens_seen": 1728774144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843232323232323, + "loss": 1.6362, + "theoretical_loss": 3.472927209415995, + "tokens_seen": 1728839680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843030303030303, + "loss": 1.2885, + "theoretical_loss": 3.4729159159802636, + "tokens_seen": 1728905216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842828282828283, + "loss": 1.5108, + "theoretical_loss": 3.4729046230924743, + "tokens_seen": 1728970752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842626262626263, + "loss": 1.5734, + "theoretical_loss": 3.472893330752579, + "tokens_seen": 1729036288 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048424242424242425, + "loss": 1.2754, + "theoretical_loss": 3.4728820389605306, + "tokens_seen": 1729101824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842222222222222, + "loss": 1.3543, + "theoretical_loss": 3.4728707477162817, + "tokens_seen": 1729167360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842020202020202, + "loss": 1.4882, + "theoretical_loss": 3.472859457019785, + "tokens_seen": 1729232896 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048418181818181817, + "loss": 1.288, + "theoretical_loss": 3.4728481668709934, + "tokens_seen": 1729298432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841616161616162, + "loss": 1.1096, + "theoretical_loss": 3.4728368772698586, + "tokens_seen": 1729363968 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048414141414141414, + "loss": 1.3439, + "theoretical_loss": 3.4728255882163346, + "tokens_seen": 1729429504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048412121212121215, + "loss": 1.3121, + "theoretical_loss": 3.4728142997103735, + "tokens_seen": 1729495040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841010101010101, + "loss": 1.3209, + "theoretical_loss": 3.472803011751928, + "tokens_seen": 1729560576 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048408080808080807, + "loss": 1.3692, + "theoretical_loss": 3.472791724340951, + "tokens_seen": 1729626112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840606060606061, + "loss": 1.5298, + "theoretical_loss": 3.472780437477395, + "tokens_seen": 1729691648 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048404040404040403, + "loss": 1.4671, + "theoretical_loss": 3.472769151161213, + "tokens_seen": 1729757184 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048402020202020204, + "loss": 1.5067, + "theoretical_loss": 3.4727578653923565, + "tokens_seen": 1729822720 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484, + "loss": 1.5815, + "theoretical_loss": 3.4727465801707798, + "tokens_seen": 1729888256 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048397979797979796, + "loss": 1.5065, + "theoretical_loss": 3.472735295496435, + "tokens_seen": 1729953792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048395959595959597, + "loss": 1.3621, + "theoretical_loss": 3.4727240113692748, + "tokens_seen": 1730019328 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483939393939394, + "loss": 1.2581, + "theoretical_loss": 3.472712727789252, + "tokens_seen": 1730084864 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048391919191919194, + "loss": 1.4585, + "theoretical_loss": 3.4727014447563196, + "tokens_seen": 1730150400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838989898989899, + "loss": 1.3508, + "theoretical_loss": 3.472690162270429, + "tokens_seen": 1730215936 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048387878787878785, + "loss": 1.4799, + "theoretical_loss": 3.472678880331535, + "tokens_seen": 1730281472 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048385858585858586, + "loss": 1.3777, + "theoretical_loss": 3.4726675989395894, + "tokens_seen": 1730347008 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.46120479702949524, + "objective/train/avg_token_score": 0.012367086485028267, + "objective/train/docs_used": 979527, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5714759826660156, + "objective/train/original_loss": 4.267512321472168, + "objective/train/theoretical_loss": 3.472656318094545, + "objective/train/tokens_used": 89271776, + "objective/train/unlikelihood_loss": 0.061104901134967804, + "theoretical_loss": 3.472656318094545, + "tokens_seen": 1730412544 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048383838383838387, + "loss": 1.3963, + "theoretical_loss": 3.472656318094545, + "tokens_seen": 1730412544 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048381818181818183, + "loss": 1.4758, + "theoretical_loss": 3.4726450377963545, + "tokens_seen": 1730478080 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837979797979798, + "loss": 1.2657, + "theoretical_loss": 3.47263375804497, + "tokens_seen": 1730543616 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048377777777777774, + "loss": 1.3777, + "theoretical_loss": 3.472622478840346, + "tokens_seen": 1730609152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837575757575758, + "loss": 1.4637, + "theoretical_loss": 3.472611200182434, + "tokens_seen": 1730674688 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048373737373737377, + "loss": 1.3547, + "theoretical_loss": 3.472599922071187, + "tokens_seen": 1730740224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837171717171717, + "loss": 1.4689, + "theoretical_loss": 3.472588644506558, + "tokens_seen": 1730805760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836969696969697, + "loss": 1.6301, + "theoretical_loss": 3.4725773674884994, + "tokens_seen": 1730871296 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048367676767676764, + "loss": 1.5216, + "theoretical_loss": 3.4725660910169647, + "tokens_seen": 1730936832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836565656565657, + "loss": 1.5067, + "theoretical_loss": 3.472554815091906, + "tokens_seen": 1731002368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048363636363636366, + "loss": 1.4398, + "theoretical_loss": 3.472543539713277, + "tokens_seen": 1731067904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836161616161616, + "loss": 1.2717, + "theoretical_loss": 3.4725322648810297, + "tokens_seen": 1731133440 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048359595959595957, + "loss": 1.3783, + "theoretical_loss": 3.4725209905951173, + "tokens_seen": 1731198976 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048357575757575764, + "loss": 1.5397, + "theoretical_loss": 3.472509716855493, + "tokens_seen": 1731264512 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835555555555556, + "loss": 1.4858, + "theoretical_loss": 3.4724984436621087, + "tokens_seen": 1731330048 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048353535353535355, + "loss": 1.3623, + "theoretical_loss": 3.4724871710149183, + "tokens_seen": 1731395584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835151515151515, + "loss": 1.3471, + "theoretical_loss": 3.4724758989138738, + "tokens_seen": 1731461120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048349494949494946, + "loss": 1.5417, + "theoretical_loss": 3.472464627358929, + "tokens_seen": 1731526656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048347474747474753, + "loss": 1.6138, + "theoretical_loss": 3.4724533563500355, + "tokens_seen": 1731592192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834545454545455, + "loss": 1.6054, + "theoretical_loss": 3.472442085887147, + "tokens_seen": 1731657728 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048343434343434344, + "loss": 1.6946, + "theoretical_loss": 3.472430815970217, + "tokens_seen": 1731723264 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834141414141414, + "loss": 1.4545, + "theoretical_loss": 3.4724195465991974, + "tokens_seen": 1731788800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833939393939394, + "loss": 1.4118, + "theoretical_loss": 3.4724082777740413, + "tokens_seen": 1731854336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833737373737374, + "loss": 1.2402, + "theoretical_loss": 3.472397009494702, + "tokens_seen": 1731919872 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833535353535354, + "loss": 1.5217, + "theoretical_loss": 3.4723857417611317, + "tokens_seen": 1731985408 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.37738269567489624, + "objective/train/avg_token_score": 0.007554531563073397, + "objective/train/docs_used": 980288, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1984022855758667, + "objective/train/original_loss": 4.563331127166748, + "objective/train/theoretical_loss": 3.4723744745732836, + "objective/train/tokens_used": 90910176, + "objective/train/unlikelihood_loss": 0.06496313959360123, + "theoretical_loss": 3.4723744745732836, + "tokens_seen": 1732050944 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048333333333333334, + "loss": 1.1324, + "theoretical_loss": 3.4723744745732836, + "tokens_seen": 1732050944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833131313131313, + "loss": 1.2802, + "theoretical_loss": 3.472363207931111, + "tokens_seen": 1732116480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832929292929293, + "loss": 1.3541, + "theoretical_loss": 3.4723519418345665, + "tokens_seen": 1732182016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832727272727273, + "loss": 1.5818, + "theoretical_loss": 3.4723406762836033, + "tokens_seen": 1732247552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048325252525252527, + "loss": 1.4111, + "theoretical_loss": 3.472329411278174, + "tokens_seen": 1732313088 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048323232323232323, + "loss": 1.4464, + "theoretical_loss": 3.472318146818232, + "tokens_seen": 1732378624 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048321212121212124, + "loss": 1.4059, + "theoretical_loss": 3.4723068829037294, + "tokens_seen": 1732444160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831919191919192, + "loss": 1.2797, + "theoretical_loss": 3.47229561953462, + "tokens_seen": 1732509696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831717171717172, + "loss": 1.3734, + "theoretical_loss": 3.4722843567108566, + "tokens_seen": 1732575232 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048315151515151516, + "loss": 1.3392, + "theoretical_loss": 3.4722730944323916, + "tokens_seen": 1732640768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831313131313131, + "loss": 1.4454, + "theoretical_loss": 3.4722618326991785, + "tokens_seen": 1732706304 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048311111111111113, + "loss": 1.578, + "theoretical_loss": 3.4722505715111707, + "tokens_seen": 1732771840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830909090909091, + "loss": 1.3528, + "theoretical_loss": 3.4722393108683205, + "tokens_seen": 1732837376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830707070707071, + "loss": 1.419, + "theoretical_loss": 3.472228050770581, + "tokens_seen": 1732902912 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048305050505050506, + "loss": 1.3326, + "theoretical_loss": 3.472216791217905, + "tokens_seen": 1732968448 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048303030303030307, + "loss": 1.6869, + "theoretical_loss": 3.4722055322102463, + "tokens_seen": 1733033984 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483010101010101, + "loss": 1.5042, + "theoretical_loss": 3.472194273747557, + "tokens_seen": 1733099520 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482989898989899, + "loss": 1.3897, + "theoretical_loss": 3.47218301582979, + "tokens_seen": 1733165056 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482969696969697, + "loss": 1.2143, + "theoretical_loss": 3.4721717584569003, + "tokens_seen": 1733230592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048294949494949495, + "loss": 1.5147, + "theoretical_loss": 3.472160501628838, + "tokens_seen": 1733296128 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048292929292929296, + "loss": 1.3647, + "theoretical_loss": 3.4721492453455585, + "tokens_seen": 1733361664 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829090909090909, + "loss": 1.1661, + "theoretical_loss": 3.4721379896070133, + "tokens_seen": 1733427200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828888888888889, + "loss": 1.5285, + "theoretical_loss": 3.4721267344131563, + "tokens_seen": 1733492736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828686868686869, + "loss": 1.4618, + "theoretical_loss": 3.4721154797639406, + "tokens_seen": 1733558272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828484848484849, + "loss": 1.5015, + "theoretical_loss": 3.4721042256593186, + "tokens_seen": 1733623808 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.5816226601600647, + "objective/train/avg_token_score": 0.014024200849235058, + "objective/train/docs_used": 981887, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.1395461559295654, + "objective/train/original_loss": 4.308298587799072, + "objective/train/theoretical_loss": 3.472092972099244, + "objective/train/tokens_used": 92548576, + "objective/train/unlikelihood_loss": 0.05324159935116768, + "theoretical_loss": 3.472092972099244, + "tokens_seen": 1733689344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048282828282828285, + "loss": 1.4546, + "theoretical_loss": 3.472092972099244, + "tokens_seen": 1733689344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828080808080808, + "loss": 1.1628, + "theoretical_loss": 3.4720817190836692, + "tokens_seen": 1733754880 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048278787878787877, + "loss": 1.2467, + "theoretical_loss": 3.472070466612548, + "tokens_seen": 1733820416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827676767676768, + "loss": 1.4641, + "theoretical_loss": 3.4720592146858333, + "tokens_seen": 1733885952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827474747474748, + "loss": 1.5125, + "theoretical_loss": 3.4720479633034778, + "tokens_seen": 1733951488 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048272727272727275, + "loss": 1.3324, + "theoretical_loss": 3.472036712465435, + "tokens_seen": 1734017024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827070707070707, + "loss": 1.3651, + "theoretical_loss": 3.4720254621716578, + "tokens_seen": 1734082560 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048268686868686866, + "loss": 1.4273, + "theoretical_loss": 3.472014212422099, + "tokens_seen": 1734148096 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048266666666666667, + "loss": 1.2076, + "theoretical_loss": 3.4720029632167124, + "tokens_seen": 1734213632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826464646464647, + "loss": 1.1455, + "theoretical_loss": 3.471991714555451, + "tokens_seen": 1734279168 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048262626262626264, + "loss": 1.5309, + "theoretical_loss": 3.471980466438267, + "tokens_seen": 1734344704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826060606060606, + "loss": 1.5117, + "theoretical_loss": 3.471969218865115, + "tokens_seen": 1734410240 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048258585858585855, + "loss": 1.5216, + "theoretical_loss": 3.4719579718359466, + "tokens_seen": 1734475776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825656565656566, + "loss": 1.52, + "theoretical_loss": 3.4719467253507164, + "tokens_seen": 1734541312 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825454545454546, + "loss": 1.5335, + "theoretical_loss": 3.471935479409376, + "tokens_seen": 1734606848 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048252525252525253, + "loss": 1.531, + "theoretical_loss": 3.4719242340118797, + "tokens_seen": 1734672384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825050505050505, + "loss": 1.3912, + "theoretical_loss": 3.471912989158181, + "tokens_seen": 1734737920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048248484848484845, + "loss": 1.5328, + "theoretical_loss": 3.471901744848231, + "tokens_seen": 1734803456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824646464646465, + "loss": 1.4363, + "theoretical_loss": 3.471890501081985, + "tokens_seen": 1734868992 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048244444444444447, + "loss": 1.6167, + "theoretical_loss": 3.4718792578593956, + "tokens_seen": 1734934528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824242424242424, + "loss": 1.3632, + "theoretical_loss": 3.4718680151804158, + "tokens_seen": 1735000064 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824040404040404, + "loss": 1.3851, + "theoretical_loss": 3.4718567730449985, + "tokens_seen": 1735065600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823838383838384, + "loss": 1.2801, + "theoretical_loss": 3.4718455314530976, + "tokens_seen": 1735131136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823636363636364, + "loss": 1.3273, + "theoretical_loss": 3.471834290404665, + "tokens_seen": 1735196672 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048234343434343436, + "loss": 1.5982, + "theoretical_loss": 3.4718230498996556, + "tokens_seen": 1735262208 + }, + { + "epoch": 0.04, + "objective/train/aligned_token_count": 0.25384896993637085, + "objective/train/avg_token_score": 0.01493894774466753, + "objective/train/docs_used": 982466, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8242332339286804, + "objective/train/original_loss": 4.889101505279541, + "objective/train/theoretical_loss": 3.4718118099380213, + "objective/train/tokens_used": 94186976, + "objective/train/unlikelihood_loss": 0.03157869726419449, + "theoretical_loss": 3.4718118099380213, + "tokens_seen": 1735327744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823232323232323, + "loss": 1.1415, + "theoretical_loss": 3.4718118099380213, + "tokens_seen": 1735327744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004823030303030303, + "loss": 1.321, + "theoretical_loss": 3.4718005705197164, + "tokens_seen": 1735393280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822828282828283, + "loss": 1.6192, + "theoretical_loss": 3.4717893316446933, + "tokens_seen": 1735458816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822626262626263, + "loss": 1.277, + "theoretical_loss": 3.471778093312905, + "tokens_seen": 1735524352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048224242424242425, + "loss": 1.3975, + "theoretical_loss": 3.4717668555243053, + "tokens_seen": 1735589888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822222222222222, + "loss": 1.5485, + "theoretical_loss": 3.4717556182788476, + "tokens_seen": 1735655424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822020202020202, + "loss": 1.1766, + "theoretical_loss": 3.471744381576485, + "tokens_seen": 1735720960 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821818181818182, + "loss": 1.3168, + "theoretical_loss": 3.4717331454171703, + "tokens_seen": 1735786496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821616161616162, + "loss": 1.4985, + "theoretical_loss": 3.471721909800857, + "tokens_seen": 1735852032 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048214141414141415, + "loss": 1.4077, + "theoretical_loss": 3.4717106747274986, + "tokens_seen": 1735917568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821212121212121, + "loss": 1.4942, + "theoretical_loss": 3.471699440197048, + "tokens_seen": 1735983104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821010101010101, + "loss": 1.5322, + "theoretical_loss": 3.4716882062094587, + "tokens_seen": 1736048640 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048208080808080807, + "loss": 1.4188, + "theoretical_loss": 3.4716769727646835, + "tokens_seen": 1736114176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820606060606061, + "loss": 1.5154, + "theoretical_loss": 3.4716657398626767, + "tokens_seen": 1736179712 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048204040404040404, + "loss": 1.2208, + "theoretical_loss": 3.471654507503391, + "tokens_seen": 1736245248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048202020202020205, + "loss": 1.524, + "theoretical_loss": 3.4716432756867794, + "tokens_seen": 1736310784 + }, + { + "epoch": 0.05, + "learning_rate": 0.000482, + "loss": 1.5759, + "theoretical_loss": 3.4716320444127957, + "tokens_seen": 1736376320 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048197979797979796, + "loss": 1.3482, + "theoretical_loss": 3.471620813681393, + "tokens_seen": 1736441856 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481959595959596, + "loss": 1.4318, + "theoretical_loss": 3.4716095834925245, + "tokens_seen": 1736507392 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048193939393939393, + "loss": 1.423, + "theoretical_loss": 3.4715983538461432, + "tokens_seen": 1736572928 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048191919191919194, + "loss": 1.4259, + "theoretical_loss": 3.4715871247422037, + "tokens_seen": 1736638464 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818989898989899, + "loss": 1.6113, + "theoretical_loss": 3.4715758961806573, + "tokens_seen": 1736704000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818787878787879, + "loss": 1.4458, + "theoretical_loss": 3.4715646681614594, + "tokens_seen": 1736769536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048185858585858587, + "loss": 1.1604, + "theoretical_loss": 3.4715534406845627, + "tokens_seen": 1736835072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818383838383839, + "loss": 1.3382, + "theoretical_loss": 3.4715422137499194, + "tokens_seen": 1736900608 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.43413978815078735, + "objective/train/avg_token_score": 0.012802019715309143, + "objective/train/docs_used": 983872, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4975614547729492, + "objective/train/original_loss": 4.492546081542969, + "objective/train/theoretical_loss": 3.4715309873574847, + "objective/train/tokens_used": 95825376, + "objective/train/unlikelihood_loss": 0.04530109092593193, + "theoretical_loss": 3.4715309873574847, + "tokens_seen": 1736966144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048181818181818184, + "loss": 1.3615, + "theoretical_loss": 3.4715309873574847, + "tokens_seen": 1736966144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817979797979798, + "loss": 1.5586, + "theoretical_loss": 3.4715197615072104, + "tokens_seen": 1737031680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817777777777778, + "loss": 1.5499, + "theoretical_loss": 3.4715085361990505, + "tokens_seen": 1737097216 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048175757575757576, + "loss": 1.3085, + "theoretical_loss": 3.471497311432959, + "tokens_seen": 1737162752 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048173737373737377, + "loss": 1.4672, + "theoretical_loss": 3.4714860872088877, + "tokens_seen": 1737228288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048171717171717173, + "loss": 1.3109, + "theoretical_loss": 3.471474863526791, + "tokens_seen": 1737293824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816969696969697, + "loss": 1.5911, + "theoretical_loss": 3.471463640386623, + "tokens_seen": 1737359360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816767676767677, + "loss": 1.4829, + "theoretical_loss": 3.4714524177883352, + "tokens_seen": 1737424896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816565656565657, + "loss": 1.3043, + "theoretical_loss": 3.4714411957318827, + "tokens_seen": 1737490432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048163636363636366, + "loss": 1.5117, + "theoretical_loss": 3.4714299742172186, + "tokens_seen": 1737555968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816161616161616, + "loss": 1.5361, + "theoretical_loss": 3.4714187532442953, + "tokens_seen": 1737621504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815959595959596, + "loss": 1.4369, + "theoretical_loss": 3.471407532813067, + "tokens_seen": 1737687040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815757575757576, + "loss": 1.3458, + "theoretical_loss": 3.4713963129234875, + "tokens_seen": 1737752576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815555555555556, + "loss": 1.4904, + "theoretical_loss": 3.4713850935755093, + "tokens_seen": 1737818112 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048153535353535356, + "loss": 1.3929, + "theoretical_loss": 3.4713738747690863, + "tokens_seen": 1737883648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815151515151515, + "loss": 1.5227, + "theoretical_loss": 3.471362656504172, + "tokens_seen": 1737949184 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048149494949494947, + "loss": 1.4509, + "theoretical_loss": 3.4713514387807196, + "tokens_seen": 1738014720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048147474747474754, + "loss": 1.3549, + "theoretical_loss": 3.471340221598683, + "tokens_seen": 1738080256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004814545454545455, + "loss": 1.4395, + "theoretical_loss": 3.4713290049580157, + "tokens_seen": 1738145792 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048143434343434345, + "loss": 1.484, + "theoretical_loss": 3.4713177888586704, + "tokens_seen": 1738211328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004814141414141414, + "loss": 1.3216, + "theoretical_loss": 3.471306573300601, + "tokens_seen": 1738276864 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048139393939393936, + "loss": 1.6081, + "theoretical_loss": 3.4712953582837613, + "tokens_seen": 1738342400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048137373737373743, + "loss": 1.5769, + "theoretical_loss": 3.471284143808104, + "tokens_seen": 1738407936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813535353535354, + "loss": 1.4718, + "theoretical_loss": 3.4712729298735834, + "tokens_seen": 1738473472 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048133333333333334, + "loss": 1.3028, + "theoretical_loss": 3.471261716480152, + "tokens_seen": 1738539008 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.48277124762535095, + "objective/train/avg_token_score": 0.009705035015940666, + "objective/train/docs_used": 984348, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.655248999595642, + "objective/train/original_loss": 4.150419235229492, + "objective/train/theoretical_loss": 3.4712505036277648, + "objective/train/tokens_used": 97463776, + "objective/train/unlikelihood_loss": 0.052979037165641785, + "theoretical_loss": 3.4712505036277648, + "tokens_seen": 1738604544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813131313131313, + "loss": 1.4086, + "theoretical_loss": 3.4712505036277648, + "tokens_seen": 1738604544 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048129292929292926, + "loss": 1.3672, + "theoretical_loss": 3.4712392913163734, + "tokens_seen": 1738670080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812727272727273, + "loss": 1.4242, + "theoretical_loss": 3.4712280795459334, + "tokens_seen": 1738735616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812525252525253, + "loss": 1.5003, + "theoretical_loss": 3.4712168683163966, + "tokens_seen": 1738801152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048123232323232323, + "loss": 1.3127, + "theoretical_loss": 3.4712056576277175, + "tokens_seen": 1738866688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812121212121212, + "loss": 1.3313, + "theoretical_loss": 3.4711944474798493, + "tokens_seen": 1738932224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811919191919192, + "loss": 1.3412, + "theoretical_loss": 3.4711832378727454, + "tokens_seen": 1738997760 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811717171717172, + "loss": 1.3733, + "theoretical_loss": 3.4711720288063592, + "tokens_seen": 1739063296 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048115151515151517, + "loss": 1.4347, + "theoretical_loss": 3.4711608202806445, + "tokens_seen": 1739128832 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048113131313131313, + "loss": 1.1575, + "theoretical_loss": 3.471149612295555, + "tokens_seen": 1739194368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811111111111111, + "loss": 1.4967, + "theoretical_loss": 3.4711384048510445, + "tokens_seen": 1739259904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810909090909091, + "loss": 1.5965, + "theoretical_loss": 3.471127197947066, + "tokens_seen": 1739325440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810707070707071, + "loss": 1.6453, + "theoretical_loss": 3.4711159915835728, + "tokens_seen": 1739390976 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048105050505050506, + "loss": 1.4245, + "theoretical_loss": 3.4711047857605193, + "tokens_seen": 1739456512 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481030303030303, + "loss": 1.5783, + "theoretical_loss": 3.4710935804778584, + "tokens_seen": 1739522048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048101010101010103, + "loss": 1.2761, + "theoretical_loss": 3.471082375735544, + "tokens_seen": 1739587584 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480989898989899, + "loss": 1.4248, + "theoretical_loss": 3.47107117153353, + "tokens_seen": 1739653120 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480969696969697, + "loss": 1.6432, + "theoretical_loss": 3.471059967871769, + "tokens_seen": 1739718656 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048094949494949496, + "loss": 1.394, + "theoretical_loss": 3.4710487647502157, + "tokens_seen": 1739784192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004809292929292929, + "loss": 1.3901, + "theoretical_loss": 3.4710375621688234, + "tokens_seen": 1739849728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004809090909090909, + "loss": 1.4413, + "theoretical_loss": 3.471026360127545, + "tokens_seen": 1739915264 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808888888888889, + "loss": 1.4342, + "theoretical_loss": 3.471015158626335, + "tokens_seen": 1739980800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808686868686869, + "loss": 1.2352, + "theoretical_loss": 3.471003957665147, + "tokens_seen": 1740046336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048084848484848485, + "loss": 1.2348, + "theoretical_loss": 3.470992757243934, + "tokens_seen": 1740111872 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048082828282828286, + "loss": 1.3523, + "theoretical_loss": 3.4709815573626495, + "tokens_seen": 1740177408 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.33125612139701843, + "objective/train/avg_token_score": 0.017105352133512497, + "objective/train/docs_used": 985100, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2295209169387817, + "objective/train/original_loss": 4.453027248382568, + "objective/train/theoretical_loss": 3.4709703580212485, + "objective/train/tokens_used": 99102176, + "objective/train/unlikelihood_loss": 0.0723608136177063, + "theoretical_loss": 3.4709703580212485, + "tokens_seen": 1740242944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808080808080808, + "loss": 1.4608, + "theoretical_loss": 3.4709703580212485, + "tokens_seen": 1740242944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807878787878788, + "loss": 1.4108, + "theoretical_loss": 3.470959159219683, + "tokens_seen": 1740308480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807676767676768, + "loss": 1.5332, + "theoretical_loss": 3.4709479609579077, + "tokens_seen": 1740374016 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048074747474747474, + "loss": 1.7023, + "theoretical_loss": 3.470936763235876, + "tokens_seen": 1740439552 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048072727272727275, + "loss": 1.24, + "theoretical_loss": 3.470925566053541, + "tokens_seen": 1740505088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807070707070707, + "loss": 1.4697, + "theoretical_loss": 3.4709143694108575, + "tokens_seen": 1740570624 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048068686868686867, + "loss": 1.4578, + "theoretical_loss": 3.4709031733077786, + "tokens_seen": 1740636160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806666666666667, + "loss": 1.5702, + "theoretical_loss": 3.4708919777442575, + "tokens_seen": 1740701696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806464646464647, + "loss": 1.5591, + "theoretical_loss": 3.470880782720249, + "tokens_seen": 1740767232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048062626262626264, + "loss": 1.2108, + "theoretical_loss": 3.470869588235705, + "tokens_seen": 1740832768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806060606060606, + "loss": 1.3671, + "theoretical_loss": 3.4708583942905813, + "tokens_seen": 1740898304 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048058585858585856, + "loss": 1.5984, + "theoretical_loss": 3.4708472008848297, + "tokens_seen": 1740963840 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048056565656565657, + "loss": 1.5255, + "theoretical_loss": 3.4708360080184057, + "tokens_seen": 1741029376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805454545454546, + "loss": 1.5005, + "theoretical_loss": 3.4708248156912616, + "tokens_seen": 1741094912 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048052525252525254, + "loss": 1.5318, + "theoretical_loss": 3.470813623903352, + "tokens_seen": 1741160448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805050505050505, + "loss": 1.3555, + "theoretical_loss": 3.47080243265463, + "tokens_seen": 1741225984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048048484848484845, + "loss": 1.324, + "theoretical_loss": 3.47079124194505, + "tokens_seen": 1741291520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804646464646465, + "loss": 1.4643, + "theoretical_loss": 3.4707800517745646, + "tokens_seen": 1741357056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804444444444445, + "loss": 1.2959, + "theoretical_loss": 3.470768862143129, + "tokens_seen": 1741422592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048042424242424243, + "loss": 1.3676, + "theoretical_loss": 3.470757673050696, + "tokens_seen": 1741488128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804040404040404, + "loss": 1.4532, + "theoretical_loss": 3.4707464844972193, + "tokens_seen": 1741553664 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048038383838383834, + "loss": 1.3073, + "theoretical_loss": 3.470735296482653, + "tokens_seen": 1741619200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803636363636364, + "loss": 1.3783, + "theoretical_loss": 3.470724109006951, + "tokens_seen": 1741684736 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048034343434343437, + "loss": 1.6342, + "theoretical_loss": 3.470712922070067, + "tokens_seen": 1741750272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803232323232323, + "loss": 1.5602, + "theoretical_loss": 3.4707017356719545, + "tokens_seen": 1741815808 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.4005987346172333, + "objective/train/avg_token_score": 0.019556434825062752, + "objective/train/docs_used": 986266, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.536611795425415, + "objective/train/original_loss": 4.283116817474365, + "objective/train/theoretical_loss": 3.4706905498125673, + "objective/train/tokens_used": 100740576, + "objective/train/unlikelihood_loss": 0.08906306326389313, + "theoretical_loss": 3.4706905498125673, + "tokens_seen": 1741881344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803030303030303, + "loss": 1.4406, + "theoretical_loss": 3.4706905498125673, + "tokens_seen": 1741881344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802828282828283, + "loss": 1.4997, + "theoretical_loss": 3.47067936449186, + "tokens_seen": 1741946880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802626262626263, + "loss": 1.6152, + "theoretical_loss": 3.4706681797097847, + "tokens_seen": 1742012416 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048024242424242426, + "loss": 1.5295, + "theoretical_loss": 3.470656995466297, + "tokens_seen": 1742077952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802222222222222, + "loss": 1.3141, + "theoretical_loss": 3.47064581176135, + "tokens_seen": 1742143488 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048020202020202017, + "loss": 1.3606, + "theoretical_loss": 3.470634628594897, + "tokens_seen": 1742209024 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801818181818182, + "loss": 1.4034, + "theoretical_loss": 3.4706234459668925, + "tokens_seen": 1742274560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801616161616162, + "loss": 1.4074, + "theoretical_loss": 3.47061226387729, + "tokens_seen": 1742340096 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048014141414141415, + "loss": 1.2829, + "theoretical_loss": 3.470601082326043, + "tokens_seen": 1742405632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801212121212121, + "loss": 1.4758, + "theoretical_loss": 3.4705899013131063, + "tokens_seen": 1742471168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801010101010101, + "loss": 1.3614, + "theoretical_loss": 3.470578720838433, + "tokens_seen": 1742536704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048008080808080813, + "loss": 1.3864, + "theoretical_loss": 3.4705675409019774, + "tokens_seen": 1742602240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800606060606061, + "loss": 1.4217, + "theoretical_loss": 3.4705563615036925, + "tokens_seen": 1742667776 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048004040404040404, + "loss": 1.1732, + "theoretical_loss": 3.4705451826435327, + "tokens_seen": 1742733312 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480020202020202, + "loss": 1.4523, + "theoretical_loss": 3.4705340043214523, + "tokens_seen": 1742798848 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048, + "loss": 1.4718, + "theoretical_loss": 3.4705228265374046, + "tokens_seen": 1742864384 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479979797979798, + "loss": 1.4891, + "theoretical_loss": 3.470511649291344, + "tokens_seen": 1742929920 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479959595959596, + "loss": 1.2984, + "theoretical_loss": 3.4705004725832236, + "tokens_seen": 1742995456 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047993939393939394, + "loss": 1.5396, + "theoretical_loss": 3.470489296412998, + "tokens_seen": 1743060992 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047991919191919195, + "loss": 1.6143, + "theoretical_loss": 3.470478120780621, + "tokens_seen": 1743126528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798989898989899, + "loss": 1.3893, + "theoretical_loss": 3.4704669456860455, + "tokens_seen": 1743192064 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798787878787879, + "loss": 1.3921, + "theoretical_loss": 3.4704557711292265, + "tokens_seen": 1743257600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047985858585858587, + "loss": 1.326, + "theoretical_loss": 3.470444597110118, + "tokens_seen": 1743323136 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047983838383838383, + "loss": 1.3948, + "theoretical_loss": 3.4704334236286734, + "tokens_seen": 1743388672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047981818181818184, + "loss": 1.3807, + "theoretical_loss": 3.4704222506848463, + "tokens_seen": 1743454208 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.3908846378326416, + "objective/train/avg_token_score": 0.00899689830839634, + "objective/train/docs_used": 986906, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2534172534942627, + "objective/train/original_loss": 4.556232929229736, + "objective/train/theoretical_loss": 3.470411078278591, + "objective/train/tokens_used": 102378976, + "objective/train/unlikelihood_loss": 0.03879563882946968, + "theoretical_loss": 3.470411078278591, + "tokens_seen": 1743519744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797979797979798, + "loss": 1.5736, + "theoretical_loss": 3.470411078278591, + "tokens_seen": 1743519744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797777777777778, + "loss": 1.304, + "theoretical_loss": 3.4703999064098623, + "tokens_seen": 1743585280 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047975757575757577, + "loss": 1.4849, + "theoretical_loss": 3.470388735078613, + "tokens_seen": 1743650816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797373737373738, + "loss": 1.3432, + "theoretical_loss": 3.4703775642847967, + "tokens_seen": 1743716352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047971717171717173, + "loss": 1.1625, + "theoretical_loss": 3.4703663940283684, + "tokens_seen": 1743781888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796969696969697, + "loss": 1.6368, + "theoretical_loss": 3.470355224309282, + "tokens_seen": 1743847424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796767676767677, + "loss": 1.4579, + "theoretical_loss": 3.470344055127491, + "tokens_seen": 1743912960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047965656565656566, + "loss": 1.4415, + "theoretical_loss": 3.4703328864829492, + "tokens_seen": 1743978496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047963636363636367, + "loss": 1.5911, + "theoretical_loss": 3.4703217183756108, + "tokens_seen": 1744044032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796161616161616, + "loss": 1.2517, + "theoretical_loss": 3.47031055080543, + "tokens_seen": 1744109568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795959595959596, + "loss": 1.2024, + "theoretical_loss": 3.470299383772361, + "tokens_seen": 1744175104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795757575757576, + "loss": 1.3713, + "theoretical_loss": 3.470288217276357, + "tokens_seen": 1744240640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795555555555556, + "loss": 1.4253, + "theoretical_loss": 3.4702770513173724, + "tokens_seen": 1744306176 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047953535353535356, + "loss": 1.406, + "theoretical_loss": 3.470265885895361, + "tokens_seen": 1744371712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795151515151515, + "loss": 1.4899, + "theoretical_loss": 3.4702547210102774, + "tokens_seen": 1744437248 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794949494949495, + "loss": 1.5386, + "theoretical_loss": 3.470243556662075, + "tokens_seen": 1744502784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794747474747475, + "loss": 1.2509, + "theoretical_loss": 3.470232392850708, + "tokens_seen": 1744568320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794545454545455, + "loss": 1.7622, + "theoretical_loss": 3.470221229576131, + "tokens_seen": 1744633856 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047943434343434345, + "loss": 1.3678, + "theoretical_loss": 3.4702100668382965, + "tokens_seen": 1744699392 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794141414141414, + "loss": 1.4218, + "theoretical_loss": 3.47019890463716, + "tokens_seen": 1744764928 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047939393939393937, + "loss": 1.4213, + "theoretical_loss": 3.470187742972675, + "tokens_seen": 1744830464 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793737373737374, + "loss": 1.4785, + "theoretical_loss": 3.470176581844796, + "tokens_seen": 1744896000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793535353535354, + "loss": 1.5449, + "theoretical_loss": 3.4701654212534763, + "tokens_seen": 1744961536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047933333333333335, + "loss": 1.287, + "theoretical_loss": 3.4701542611986698, + "tokens_seen": 1745027072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793131313131313, + "loss": 1.567, + "theoretical_loss": 3.470143101680332, + "tokens_seen": 1745092608 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.2597140669822693, + "objective/train/avg_token_score": 0.04489205777645111, + "objective/train/docs_used": 988319, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.912868082523346, + "objective/train/original_loss": 4.443542003631592, + "objective/train/theoretical_loss": 3.4701319426984156, + "objective/train/tokens_used": 104017376, + "objective/train/unlikelihood_loss": 0.06317847967147827, + "theoretical_loss": 3.4701319426984156, + "tokens_seen": 1745158144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047929292929292926, + "loss": 1.2568, + "theoretical_loss": 3.4701319426984156, + "tokens_seen": 1745158144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792727272727273, + "loss": 1.3641, + "theoretical_loss": 3.4701207842528747, + "tokens_seen": 1745223680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792525252525253, + "loss": 1.5416, + "theoretical_loss": 3.470109626343664, + "tokens_seen": 1745289216 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047923232323232324, + "loss": 1.4213, + "theoretical_loss": 3.470098468970738, + "tokens_seen": 1745354752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792121212121212, + "loss": 1.0977, + "theoretical_loss": 3.470087312134049, + "tokens_seen": 1745420288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047919191919191915, + "loss": 1.5069, + "theoretical_loss": 3.470076155833553, + "tokens_seen": 1745485824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791717171717172, + "loss": 1.569, + "theoretical_loss": 3.4700650000692033, + "tokens_seen": 1745551360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791515151515152, + "loss": 1.5586, + "theoretical_loss": 3.4700538448409537, + "tokens_seen": 1745616896 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047913131313131313, + "loss": 1.5639, + "theoretical_loss": 3.470042690148759, + "tokens_seen": 1745682432 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791111111111111, + "loss": 1.4329, + "theoretical_loss": 3.4700315359925726, + "tokens_seen": 1745747968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790909090909091, + "loss": 1.4756, + "theoretical_loss": 3.470020382372349, + "tokens_seen": 1745813504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790707070707071, + "loss": 1.4192, + "theoretical_loss": 3.470009229288043, + "tokens_seen": 1745879040 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047905050505050507, + "loss": 1.2349, + "theoretical_loss": 3.469998076739607, + "tokens_seen": 1745944576 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479030303030303, + "loss": 1.4639, + "theoretical_loss": 3.469986924726997, + "tokens_seen": 1746010112 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479010101010101, + "loss": 1.5232, + "theoretical_loss": 3.469975773250166, + "tokens_seen": 1746075648 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478989898989899, + "loss": 1.1636, + "theoretical_loss": 3.4699646223090688, + "tokens_seen": 1746141184 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478969696969697, + "loss": 1.4086, + "theoretical_loss": 3.469953471903659, + "tokens_seen": 1746206720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047894949494949496, + "loss": 1.3567, + "theoretical_loss": 3.469942322033891, + "tokens_seen": 1746272256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789292929292929, + "loss": 1.6627, + "theoretical_loss": 3.4699311726997193, + "tokens_seen": 1746337792 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047890909090909093, + "loss": 1.4643, + "theoretical_loss": 3.469920023901097, + "tokens_seen": 1746403328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788888888888889, + "loss": 1.393, + "theoretical_loss": 3.46990887563798, + "tokens_seen": 1746468864 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788686868686869, + "loss": 1.6576, + "theoretical_loss": 3.469897727910321, + "tokens_seen": 1746534400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047884848484848485, + "loss": 1.3613, + "theoretical_loss": 3.4698865807180748, + "tokens_seen": 1746599936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788282828282828, + "loss": 1.5132, + "theoretical_loss": 3.4698754340611955, + "tokens_seen": 1746665472 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788080808080808, + "loss": 1.4484, + "theoretical_loss": 3.4698642879396373, + "tokens_seen": 1746731008 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.29673752188682556, + "objective/train/avg_token_score": 0.007496143225580454, + "objective/train/docs_used": 989102, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9727373123168945, + "objective/train/original_loss": 4.123487949371338, + "objective/train/theoretical_loss": 3.4698531423533545, + "objective/train/tokens_used": 105655776, + "objective/train/unlikelihood_loss": 0.07145555317401886, + "theoretical_loss": 3.4698531423533545, + "tokens_seen": 1746796544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787878787878788, + "loss": 1.2235, + "theoretical_loss": 3.4698531423533545, + "tokens_seen": 1746796544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787676767676768, + "loss": 1.4326, + "theoretical_loss": 3.469841997302301, + "tokens_seen": 1746862080 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047874747474747475, + "loss": 1.4788, + "theoretical_loss": 3.4698308527864317, + "tokens_seen": 1746927616 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047872727272727276, + "loss": 1.4569, + "theoretical_loss": 3.4698197088056997, + "tokens_seen": 1746993152 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787070707070707, + "loss": 1.3571, + "theoretical_loss": 3.4698085653600605, + "tokens_seen": 1747058688 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047868686868686867, + "loss": 1.3078, + "theoretical_loss": 3.469797422449467, + "tokens_seen": 1747124224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786666666666667, + "loss": 1.3261, + "theoretical_loss": 3.4697862800738752, + "tokens_seen": 1747189760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047864646464646464, + "loss": 1.5378, + "theoretical_loss": 3.4697751382332376, + "tokens_seen": 1747255296 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047862626262626265, + "loss": 1.3303, + "theoretical_loss": 3.46976399692751, + "tokens_seen": 1747320832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786060606060606, + "loss": 1.513, + "theoretical_loss": 3.469752856156645, + "tokens_seen": 1747386368 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047858585858585856, + "loss": 1.5127, + "theoretical_loss": 3.469741715920598, + "tokens_seen": 1747451904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785656565656566, + "loss": 1.511, + "theoretical_loss": 3.4697305762193227, + "tokens_seen": 1747517440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785454545454546, + "loss": 1.3687, + "theoretical_loss": 3.469719437052774, + "tokens_seen": 1747582976 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047852525252525254, + "loss": 1.2167, + "theoretical_loss": 3.4697082984209056, + "tokens_seen": 1747648512 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785050505050505, + "loss": 1.3243, + "theoretical_loss": 3.469697160323672, + "tokens_seen": 1747714048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047848484848484846, + "loss": 1.7173, + "theoretical_loss": 3.469686022761028, + "tokens_seen": 1747779584 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047846464646464647, + "loss": 1.2967, + "theoretical_loss": 3.469674885732927, + "tokens_seen": 1747845120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784444444444445, + "loss": 1.3806, + "theoretical_loss": 3.4696637492393236, + "tokens_seen": 1747910656 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047842424242424244, + "loss": 1.433, + "theoretical_loss": 3.4696526132801724, + "tokens_seen": 1747976192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784040404040404, + "loss": 1.2802, + "theoretical_loss": 3.4696414778554274, + "tokens_seen": 1748041728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783838383838384, + "loss": 1.3713, + "theoretical_loss": 3.4696303429650435, + "tokens_seen": 1748107264 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783636363636364, + "loss": 1.288, + "theoretical_loss": 3.469619208608974, + "tokens_seen": 1748172800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047834343434343437, + "loss": 1.5354, + "theoretical_loss": 3.469608074787174, + "tokens_seen": 1748238336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047832323232323233, + "loss": 1.3335, + "theoretical_loss": 3.4695969414995975, + "tokens_seen": 1748303872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783030303030303, + "loss": 1.2709, + "theoretical_loss": 3.4695858087461993, + "tokens_seen": 1748369408 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.356671541929245, + "objective/train/avg_token_score": 0.015191304497420788, + "objective/train/docs_used": 990185, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0629782676696777, + "objective/train/original_loss": 4.2517290115356445, + "objective/train/theoretical_loss": 3.469574676526933, + "objective/train/tokens_used": 107294176, + "objective/train/unlikelihood_loss": 0.06087062880396843, + "theoretical_loss": 3.469574676526933, + "tokens_seen": 1748434944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782828282828283, + "loss": 1.27, + "theoretical_loss": 3.469574676526933, + "tokens_seen": 1748434944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782626262626263, + "loss": 1.3251, + "theoretical_loss": 3.469563544841754, + "tokens_seen": 1748500480 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047824242424242426, + "loss": 1.374, + "theoretical_loss": 3.4695524136906153, + "tokens_seen": 1748566016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782222222222222, + "loss": 1.3274, + "theoretical_loss": 3.4695412830734726, + "tokens_seen": 1748631552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782020202020202, + "loss": 1.5862, + "theoretical_loss": 3.46953015299028, + "tokens_seen": 1748697088 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047818181818181824, + "loss": 1.3369, + "theoretical_loss": 3.469519023440991, + "tokens_seen": 1748762624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781616161616162, + "loss": 1.5887, + "theoretical_loss": 3.4695078944255604, + "tokens_seen": 1748828160 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047814141414141416, + "loss": 1.4068, + "theoretical_loss": 3.4694967659439433, + "tokens_seen": 1748893696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781212121212121, + "loss": 1.4508, + "theoretical_loss": 3.469485637996093, + "tokens_seen": 1748959232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047810101010101007, + "loss": 1.3742, + "theoretical_loss": 3.4694745105819647, + "tokens_seen": 1749024768 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047808080808080814, + "loss": 1.255, + "theoretical_loss": 3.4694633837015125, + "tokens_seen": 1749090304 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780606060606061, + "loss": 1.3452, + "theoretical_loss": 3.469452257354691, + "tokens_seen": 1749155840 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047804040404040405, + "loss": 1.4228, + "theoretical_loss": 3.469441131541454, + "tokens_seen": 1749221376 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478020202020202, + "loss": 1.3645, + "theoretical_loss": 3.4694300062617565, + "tokens_seen": 1749286912 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047799999999999996, + "loss": 1.2568, + "theoretical_loss": 3.469418881515553, + "tokens_seen": 1749352448 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047797979797979803, + "loss": 1.2278, + "theoretical_loss": 3.4694077573027977, + "tokens_seen": 1749417984 + }, + { + "epoch": 0.05, + "learning_rate": 0.000477959595959596, + "loss": 1.3498, + "theoretical_loss": 3.469396633623445, + "tokens_seen": 1749483520 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047793939393939394, + "loss": 1.201, + "theoretical_loss": 3.4693855104774496, + "tokens_seen": 1749549056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779191919191919, + "loss": 1.4381, + "theoretical_loss": 3.4693743878647654, + "tokens_seen": 1749614592 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778989898989899, + "loss": 1.3039, + "theoretical_loss": 3.469363265785348, + "tokens_seen": 1749680128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778787878787879, + "loss": 1.6264, + "theoretical_loss": 3.46935214423915, + "tokens_seen": 1749745664 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778585858585859, + "loss": 1.6328, + "theoretical_loss": 3.4693410232261277, + "tokens_seen": 1749811200 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047783838383838384, + "loss": 1.5476, + "theoretical_loss": 3.4693299027462343, + "tokens_seen": 1749876736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778181818181818, + "loss": 1.2902, + "theoretical_loss": 3.469318782799425, + "tokens_seen": 1749942272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777979797979798, + "loss": 1.4278, + "theoretical_loss": 3.4693076633856545, + "tokens_seen": 1750007808 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.4995723366737366, + "objective/train/avg_token_score": 0.01636279560625553, + "objective/train/docs_used": 990903, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7601079940795898, + "objective/train/original_loss": 4.128569602966309, + "objective/train/theoretical_loss": 3.4692965445048767, + "objective/train/tokens_used": 108932576, + "objective/train/unlikelihood_loss": 0.05954797938466072, + "theoretical_loss": 3.4692965445048767, + "tokens_seen": 1750073344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777777777777778, + "loss": 1.532, + "theoretical_loss": 3.4692965445048767, + "tokens_seen": 1750073344 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047775757575757577, + "loss": 1.3282, + "theoretical_loss": 3.469285426157046, + "tokens_seen": 1750138880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047773737373737373, + "loss": 1.3803, + "theoretical_loss": 3.469274308342117, + "tokens_seen": 1750204416 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047771717171717174, + "loss": 1.5482, + "theoretical_loss": 3.469263191060045, + "tokens_seen": 1750269952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776969696969697, + "loss": 1.6532, + "theoretical_loss": 3.4692520743107833, + "tokens_seen": 1750335488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776767676767677, + "loss": 1.4167, + "theoretical_loss": 3.469240958094287, + "tokens_seen": 1750401024 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047765656565656566, + "loss": 1.193, + "theoretical_loss": 3.469229842410511, + "tokens_seen": 1750466560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776363636363636, + "loss": 1.3584, + "theoretical_loss": 3.4692187272594093, + "tokens_seen": 1750532096 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047761616161616163, + "loss": 1.439, + "theoretical_loss": 3.469207612640936, + "tokens_seen": 1750597632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775959595959596, + "loss": 1.2879, + "theoretical_loss": 3.469196498555047, + "tokens_seen": 1750663168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775757575757576, + "loss": 1.3754, + "theoretical_loss": 3.469185385001696, + "tokens_seen": 1750728704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047755555555555556, + "loss": 1.2025, + "theoretical_loss": 3.4691742719808376, + "tokens_seen": 1750794240 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047753535353535357, + "loss": 1.5471, + "theoretical_loss": 3.469163159492426, + "tokens_seen": 1750859776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775151515151515, + "loss": 1.2861, + "theoretical_loss": 3.469152047536416, + "tokens_seen": 1750925312 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774949494949495, + "loss": 1.4033, + "theoretical_loss": 3.4691409361127628, + "tokens_seen": 1750990848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774747474747475, + "loss": 1.3833, + "theoretical_loss": 3.46912982522142, + "tokens_seen": 1751056384 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047745454545454545, + "loss": 1.5292, + "theoretical_loss": 3.469118714862343, + "tokens_seen": 1751121920 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047743434343434346, + "loss": 1.3997, + "theoretical_loss": 3.469107605035486, + "tokens_seen": 1751187456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774141414141414, + "loss": 1.3918, + "theoretical_loss": 3.4690964957408035, + "tokens_seen": 1751252992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773939393939394, + "loss": 1.5265, + "theoretical_loss": 3.46908538697825, + "tokens_seen": 1751318528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773737373737374, + "loss": 1.2903, + "theoretical_loss": 3.4690742787477804, + "tokens_seen": 1751384064 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773535353535354, + "loss": 1.4772, + "theoretical_loss": 3.4690631710493496, + "tokens_seen": 1751449600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047733333333333335, + "loss": 1.4411, + "theoretical_loss": 3.469052063882912, + "tokens_seen": 1751515136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773131313131313, + "loss": 1.4693, + "theoretical_loss": 3.4690409572484215, + "tokens_seen": 1751580672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047729292929292927, + "loss": 1.6307, + "theoretical_loss": 3.469029851145833, + "tokens_seen": 1751646208 + }, + { + "epoch": 0.05, + "objective/train/aligned_token_count": 0.35160067677497864, + "objective/train/avg_token_score": 0.0432390458881855, + "objective/train/docs_used": 991641, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.203386664390564, + "objective/train/original_loss": 4.336913108825684, + "objective/train/theoretical_loss": 3.469018745575102, + "objective/train/tokens_used": 110570976, + "objective/train/unlikelihood_loss": 0.058832619339227676, + "theoretical_loss": 3.469018745575102, + "tokens_seen": 1751711744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772727272727273, + "loss": 1.3743, + "theoretical_loss": 3.469018745575102, + "tokens_seen": 1751711744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772525252525253, + "loss": 1.4335, + "theoretical_loss": 3.469007640536182, + "tokens_seen": 1751777280 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047723232323232325, + "loss": 1.5358, + "theoretical_loss": 3.4689965360290285, + "tokens_seen": 1751842816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772121212121212, + "loss": 1.2956, + "theoretical_loss": 3.4689854320535956, + "tokens_seen": 1751908352 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047719191919191916, + "loss": 1.5891, + "theoretical_loss": 3.4689743286098382, + "tokens_seen": 1751973888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771717171717172, + "loss": 1.5675, + "theoretical_loss": 3.468963225697711, + "tokens_seen": 1752039424 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771515151515152, + "loss": 1.3356, + "theoretical_loss": 3.4689521233171687, + "tokens_seen": 1752104960 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047713131313131314, + "loss": 1.4772, + "theoretical_loss": 3.468941021468166, + "tokens_seen": 1752170496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771111111111111, + "loss": 1.2844, + "theoretical_loss": 3.468929920150657, + "tokens_seen": 1752236032 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047709090909090905, + "loss": 1.5272, + "theoretical_loss": 3.468918819364597, + "tokens_seen": 1752301568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770707070707071, + "loss": 1.4227, + "theoretical_loss": 3.4689077191099402, + "tokens_seen": 1752367104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770505050505051, + "loss": 1.2629, + "theoretical_loss": 3.4688966193866415, + "tokens_seen": 1752432640 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047703030303030303, + "loss": 1.4331, + "theoretical_loss": 3.468885520194656, + "tokens_seen": 1752498176 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477010101010101, + "loss": 1.4719, + "theoretical_loss": 3.468874421533938, + "tokens_seen": 1752563712 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476989898989899, + "loss": 1.4844, + "theoretical_loss": 3.4688633234044426, + "tokens_seen": 1752629248 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476969696969697, + "loss": 1.6083, + "theoretical_loss": 3.468852225806123, + "tokens_seen": 1752694784 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047694949494949497, + "loss": 1.3576, + "theoretical_loss": 3.4688411287389362, + "tokens_seen": 1752760320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769292929292929, + "loss": 1.5143, + "theoretical_loss": 3.4688300322028356, + "tokens_seen": 1752825856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769090909090909, + "loss": 1.4537, + "theoretical_loss": 3.468818936197776, + "tokens_seen": 1752891392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768888888888889, + "loss": 1.5708, + "theoretical_loss": 3.4688078407237124, + "tokens_seen": 1752956928 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768686868686869, + "loss": 1.4414, + "theoretical_loss": 3.4687967457805993, + "tokens_seen": 1753022464 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047684848484848486, + "loss": 1.5843, + "theoretical_loss": 3.4687856513683917, + "tokens_seen": 1753088000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768282828282828, + "loss": 1.5725, + "theoretical_loss": 3.468774557487044, + "tokens_seen": 1753153536 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047680808080808083, + "loss": 1.2515, + "theoretical_loss": 3.4687634641365115, + "tokens_seen": 1753219072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767878787878788, + "loss": 1.7292, + "theoretical_loss": 3.4687523713167483, + "tokens_seen": 1753284608 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.5292032957077026, + "objective/train/avg_token_score": 0.022525779902935028, + "objective/train/docs_used": 992944, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7439391613006592, + "objective/train/original_loss": 3.8954503536224365, + "objective/train/theoretical_loss": 3.468741279027709, + "objective/train/tokens_used": 112209376, + "objective/train/unlikelihood_loss": 0.06822893768548965, + "theoretical_loss": 3.468741279027709, + "tokens_seen": 1753350144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767676767676768, + "loss": 1.5733, + "theoretical_loss": 3.468741279027709, + "tokens_seen": 1753350144 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047674747474747475, + "loss": 1.3831, + "theoretical_loss": 3.4687301872693497, + "tokens_seen": 1753415680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767272727272727, + "loss": 1.2289, + "theoretical_loss": 3.4687190960416245, + "tokens_seen": 1753481216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767070707070707, + "loss": 1.2174, + "theoretical_loss": 3.468708005344487, + "tokens_seen": 1753546752 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047668686868686873, + "loss": 1.4894, + "theoretical_loss": 3.468696915177894, + "tokens_seen": 1753612288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766666666666667, + "loss": 1.7117, + "theoretical_loss": 3.4686858255417987, + "tokens_seen": 1753677824 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047664646464646465, + "loss": 1.5382, + "theoretical_loss": 3.4686747364361565, + "tokens_seen": 1753743360 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047662626262626266, + "loss": 1.6347, + "theoretical_loss": 3.4686636478609225, + "tokens_seen": 1753808896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766060606060606, + "loss": 1.3016, + "theoretical_loss": 3.4686525598160514, + "tokens_seen": 1753874432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765858585858586, + "loss": 1.2402, + "theoretical_loss": 3.4686414723014973, + "tokens_seen": 1753939968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765656565656566, + "loss": 1.6342, + "theoretical_loss": 3.4686303853172165, + "tokens_seen": 1754005504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047654545454545454, + "loss": 1.4743, + "theoretical_loss": 3.468619298863162, + "tokens_seen": 1754071040 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047652525252525255, + "loss": 1.5816, + "theoretical_loss": 3.46860821293929, + "tokens_seen": 1754136576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765050505050505, + "loss": 1.3497, + "theoretical_loss": 3.4685971275455545, + "tokens_seen": 1754202112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764848484848485, + "loss": 1.4686, + "theoretical_loss": 3.4685860426819106, + "tokens_seen": 1754267648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764646464646465, + "loss": 1.5528, + "theoretical_loss": 3.468574958348314, + "tokens_seen": 1754333184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764444444444445, + "loss": 1.4893, + "theoretical_loss": 3.468563874544718, + "tokens_seen": 1754398720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047642424242424244, + "loss": 1.5118, + "theoretical_loss": 3.4685527912710787, + "tokens_seen": 1754464256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764040404040404, + "loss": 1.5134, + "theoretical_loss": 3.4685417085273507, + "tokens_seen": 1754529792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763838383838384, + "loss": 1.6654, + "theoretical_loss": 3.468530626313488, + "tokens_seen": 1754595328 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047636363636363637, + "loss": 1.5252, + "theoretical_loss": 3.468519544629447, + "tokens_seen": 1754660864 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763434343434344, + "loss": 1.2085, + "theoretical_loss": 3.468508463475181, + "tokens_seen": 1754726400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047632323232323233, + "loss": 1.4196, + "theoretical_loss": 3.4684973828506465, + "tokens_seen": 1754791936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763030303030303, + "loss": 1.6456, + "theoretical_loss": 3.468486302755797, + "tokens_seen": 1754857472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762828282828283, + "loss": 1.5107, + "theoretical_loss": 3.4684752231905875, + "tokens_seen": 1754923008 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.3735337257385254, + "objective/train/avg_token_score": 0.013170573860406876, + "objective/train/docs_used": 993452, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3061128854751587, + "objective/train/original_loss": 4.716046333312988, + "objective/train/theoretical_loss": 3.468464144154974, + "objective/train/tokens_used": 113847776, + "objective/train/unlikelihood_loss": 0.05805331468582153, + "theoretical_loss": 3.468464144154974, + "tokens_seen": 1754988544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762626262626263, + "loss": 1.1821, + "theoretical_loss": 3.468464144154974, + "tokens_seen": 1754988544 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047624242424242427, + "loss": 1.3812, + "theoretical_loss": 3.468453065648911, + "tokens_seen": 1755054080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762222222222222, + "loss": 1.5309, + "theoretical_loss": 3.468441987672353, + "tokens_seen": 1755119616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762020202020202, + "loss": 1.5255, + "theoretical_loss": 3.468430910225255, + "tokens_seen": 1755185152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761818181818182, + "loss": 1.4808, + "theoretical_loss": 3.468419833307572, + "tokens_seen": 1755250688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761616161616162, + "loss": 1.5992, + "theoretical_loss": 3.468408756919259, + "tokens_seen": 1755316224 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047614141414141416, + "loss": 1.4779, + "theoretical_loss": 3.468397681060271, + "tokens_seen": 1755381760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761212121212121, + "loss": 1.4515, + "theoretical_loss": 3.4683866057305623, + "tokens_seen": 1755447296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761010101010101, + "loss": 1.3284, + "theoretical_loss": 3.468375530930089, + "tokens_seen": 1755512832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760808080808081, + "loss": 1.4936, + "theoretical_loss": 3.468364456658805, + "tokens_seen": 1755578368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760606060606061, + "loss": 1.4289, + "theoretical_loss": 3.4683533829166664, + "tokens_seen": 1755643904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047604040404040406, + "loss": 1.5279, + "theoretical_loss": 3.468342309703627, + "tokens_seen": 1755709440 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476020202020202, + "loss": 1.1404, + "theoretical_loss": 3.4683312370196426, + "tokens_seen": 1755774976 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047599999999999997, + "loss": 1.6458, + "theoretical_loss": 3.4683201648646675, + "tokens_seen": 1755840512 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047597979797979803, + "loss": 1.3894, + "theoretical_loss": 3.4683090932386573, + "tokens_seen": 1755906048 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475959595959596, + "loss": 1.5657, + "theoretical_loss": 3.4682980221415662, + "tokens_seen": 1755971584 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047593939393939395, + "loss": 1.2969, + "theoretical_loss": 3.46828695157335, + "tokens_seen": 1756037120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759191919191919, + "loss": 1.4629, + "theoretical_loss": 3.4682758815339634, + "tokens_seen": 1756102656 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047589898989898986, + "loss": 1.4144, + "theoretical_loss": 3.4682648120233615, + "tokens_seen": 1756168192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047587878787878793, + "loss": 1.2853, + "theoretical_loss": 3.4682537430414992, + "tokens_seen": 1756233728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758585858585859, + "loss": 1.1362, + "theoretical_loss": 3.468242674588331, + "tokens_seen": 1756299264 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047583838383838384, + "loss": 1.4003, + "theoretical_loss": 3.468231606663813, + "tokens_seen": 1756364800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758181818181818, + "loss": 1.3655, + "theoretical_loss": 3.4682205392678993, + "tokens_seen": 1756430336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757979797979798, + "loss": 1.2837, + "theoretical_loss": 3.4682094724005457, + "tokens_seen": 1756495872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757777777777778, + "loss": 1.2286, + "theoretical_loss": 3.4681984060617066, + "tokens_seen": 1756561408 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.4603494703769684, + "objective/train/avg_token_score": 0.007801194675266743, + "objective/train/docs_used": 994709, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4813201427459717, + "objective/train/original_loss": 3.937505006790161, + "objective/train/theoretical_loss": 3.4681873402513372, + "objective/train/tokens_used": 115486176, + "objective/train/unlikelihood_loss": 0.07865725457668304, + "theoretical_loss": 3.4681873402513372, + "tokens_seen": 1756626944 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757575757575758, + "loss": 1.5432, + "theoretical_loss": 3.4681873402513372, + "tokens_seen": 1756626944 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047573737373737373, + "loss": 1.328, + "theoretical_loss": 3.4681762749693927, + "tokens_seen": 1756692480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757171717171717, + "loss": 1.3619, + "theoretical_loss": 3.4681652102158282, + "tokens_seen": 1756758016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756969696969697, + "loss": 1.2638, + "theoretical_loss": 3.4681541459905985, + "tokens_seen": 1756823552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756767676767677, + "loss": 1.5705, + "theoretical_loss": 3.468143082293659, + "tokens_seen": 1756889088 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047565656565656567, + "loss": 1.3297, + "theoretical_loss": 3.4681320191249645, + "tokens_seen": 1756954624 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756363636363636, + "loss": 1.2614, + "theoretical_loss": 3.46812095648447, + "tokens_seen": 1757020160 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047561616161616164, + "loss": 1.3043, + "theoretical_loss": 3.468109894372131, + "tokens_seen": 1757085696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755959595959596, + "loss": 1.4471, + "theoretical_loss": 3.468098832787902, + "tokens_seen": 1757151232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755757575757576, + "loss": 1.4198, + "theoretical_loss": 3.4680877717317387, + "tokens_seen": 1757216768 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047555555555555556, + "loss": 1.4285, + "theoretical_loss": 3.4680767112035955, + "tokens_seen": 1757282304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755353535353535, + "loss": 1.3098, + "theoretical_loss": 3.4680656512034282, + "tokens_seen": 1757347840 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047551515151515153, + "loss": 1.4135, + "theoretical_loss": 3.4680545917311916, + "tokens_seen": 1757413376 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754949494949495, + "loss": 1.2951, + "theoretical_loss": 3.468043532786841, + "tokens_seen": 1757478912 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754747474747475, + "loss": 1.4952, + "theoretical_loss": 3.4680324743703315, + "tokens_seen": 1757544448 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047545454545454545, + "loss": 1.3478, + "theoretical_loss": 3.4680214164816174, + "tokens_seen": 1757609984 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047543434343434347, + "loss": 1.4044, + "theoretical_loss": 3.468010359120655, + "tokens_seen": 1757675520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754141414141414, + "loss": 1.3726, + "theoretical_loss": 3.467999302287399, + "tokens_seen": 1757741056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753939393939394, + "loss": 1.5165, + "theoretical_loss": 3.4679882459818043, + "tokens_seen": 1757806592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753737373737374, + "loss": 1.6779, + "theoretical_loss": 3.4679771902038263, + "tokens_seen": 1757872128 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047535353535353535, + "loss": 1.3127, + "theoretical_loss": 3.4679661349534197, + "tokens_seen": 1757937664 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047533333333333336, + "loss": 1.4477, + "theoretical_loss": 3.4679550802305408, + "tokens_seen": 1758003200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753131313131313, + "loss": 1.3832, + "theoretical_loss": 3.467944026035144, + "tokens_seen": 1758068736 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047529292929292927, + "loss": 1.3038, + "theoretical_loss": 3.4679329723671835, + "tokens_seen": 1758134272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752727272727273, + "loss": 1.3008, + "theoretical_loss": 3.467921919226616, + "tokens_seen": 1758199808 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.37341153621673584, + "objective/train/avg_token_score": 0.01459649857133627, + "objective/train/docs_used": 995465, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.182531476020813, + "objective/train/original_loss": 4.433279991149902, + "objective/train/theoretical_loss": 3.4679108666133964, + "objective/train/tokens_used": 117124576, + "objective/train/unlikelihood_loss": 0.05860118195414543, + "theoretical_loss": 3.4679108666133964, + "tokens_seen": 1758265344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752525252525253, + "loss": 1.3201, + "theoretical_loss": 3.4679108666133964, + "tokens_seen": 1758265344 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047523232323232325, + "loss": 1.4253, + "theoretical_loss": 3.4678998145274793, + "tokens_seen": 1758330880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752121212121212, + "loss": 1.2983, + "theoretical_loss": 3.467888762968821, + "tokens_seen": 1758396416 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047519191919191916, + "loss": 1.4404, + "theoretical_loss": 3.467877711937375, + "tokens_seen": 1758461952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751717171717172, + "loss": 1.4583, + "theoretical_loss": 3.4678666614330975, + "tokens_seen": 1758527488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751515151515152, + "loss": 1.4141, + "theoretical_loss": 3.467855611455944, + "tokens_seen": 1758593024 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047513131313131314, + "loss": 1.527, + "theoretical_loss": 3.467844562005869, + "tokens_seen": 1758658560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751111111111111, + "loss": 1.3646, + "theoretical_loss": 3.4678335130828284, + "tokens_seen": 1758724096 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047509090909090906, + "loss": 1.2391, + "theoretical_loss": 3.4678224646867766, + "tokens_seen": 1758789632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750707070707071, + "loss": 1.281, + "theoretical_loss": 3.4678114168176695, + "tokens_seen": 1758855168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750505050505051, + "loss": 1.392, + "theoretical_loss": 3.467800369475462, + "tokens_seen": 1758920704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047503030303030304, + "loss": 1.2578, + "theoretical_loss": 3.46778932266011, + "tokens_seen": 1758986240 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475010101010101, + "loss": 1.4468, + "theoretical_loss": 3.467778276371568, + "tokens_seen": 1759051776 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474989898989899, + "loss": 1.3945, + "theoretical_loss": 3.467767230609791, + "tokens_seen": 1759117312 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474969696969697, + "loss": 1.166, + "theoretical_loss": 3.4677561853747356, + "tokens_seen": 1759182848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047494949494949497, + "loss": 1.5393, + "theoretical_loss": 3.4677451406663558, + "tokens_seen": 1759248384 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047492929292929293, + "loss": 1.6032, + "theoretical_loss": 3.467734096484607, + "tokens_seen": 1759313920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749090909090909, + "loss": 1.4508, + "theoretical_loss": 3.467723052829445, + "tokens_seen": 1759379456 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047488888888888895, + "loss": 1.5765, + "theoretical_loss": 3.4677120097008247, + "tokens_seen": 1759444992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748686868686869, + "loss": 1.3183, + "theoretical_loss": 3.4677009670987013, + "tokens_seen": 1759510528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047484848484848487, + "loss": 1.3187, + "theoretical_loss": 3.4676899250230306, + "tokens_seen": 1759576064 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748282828282828, + "loss": 1.3925, + "theoretical_loss": 3.4676788834737677, + "tokens_seen": 1759641600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748080808080808, + "loss": 1.4587, + "theoretical_loss": 3.467667842450868, + "tokens_seen": 1759707136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047478787878787884, + "loss": 1.4543, + "theoretical_loss": 3.467656801954286, + "tokens_seen": 1759772672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747676767676768, + "loss": 1.3954, + "theoretical_loss": 3.4676457619839773, + "tokens_seen": 1759838208 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.3027859330177307, + "objective/train/avg_token_score": 0.026031043380498886, + "objective/train/docs_used": 996601, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0335179567337036, + "objective/train/original_loss": 4.650774955749512, + "objective/train/theoretical_loss": 3.4676347225398985, + "objective/train/tokens_used": 118762976, + "objective/train/unlikelihood_loss": 0.07695260643959045, + "theoretical_loss": 3.4676347225398985, + "tokens_seen": 1759903744 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047474747474747476, + "loss": 1.1674, + "theoretical_loss": 3.4676347225398985, + "tokens_seen": 1759903744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747272727272727, + "loss": 1.283, + "theoretical_loss": 3.467623683622003, + "tokens_seen": 1759969280 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047470707070707067, + "loss": 1.4078, + "theoretical_loss": 3.467612645230248, + "tokens_seen": 1760034816 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047468686868686874, + "loss": 1.3631, + "theoretical_loss": 3.4676016073645872, + "tokens_seen": 1760100352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746666666666667, + "loss": 1.4091, + "theoretical_loss": 3.467590570024977, + "tokens_seen": 1760165888 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047464646464646465, + "loss": 1.4418, + "theoretical_loss": 3.4675795332113726, + "tokens_seen": 1760231424 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746262626262626, + "loss": 1.3409, + "theoretical_loss": 3.467568496923729, + "tokens_seen": 1760296960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746060606060606, + "loss": 1.518, + "theoretical_loss": 3.4675574611620013, + "tokens_seen": 1760362496 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047458585858585863, + "loss": 1.4428, + "theoretical_loss": 3.4675464259261455, + "tokens_seen": 1760428032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745656565656566, + "loss": 1.3315, + "theoretical_loss": 3.467535391216116, + "tokens_seen": 1760493568 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047454545454545454, + "loss": 1.5314, + "theoretical_loss": 3.46752435703187, + "tokens_seen": 1760559104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745252525252525, + "loss": 1.5786, + "theoretical_loss": 3.4675133233733613, + "tokens_seen": 1760624640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745050505050505, + "loss": 1.5052, + "theoretical_loss": 3.467502290240546, + "tokens_seen": 1760690176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744848484848485, + "loss": 1.5072, + "theoretical_loss": 3.4674912576333785, + "tokens_seen": 1760755712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744646464646465, + "loss": 1.3799, + "theoretical_loss": 3.4674802255518156, + "tokens_seen": 1760821248 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047444444444444444, + "loss": 1.359, + "theoretical_loss": 3.467469193995812, + "tokens_seen": 1760886784 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047442424242424245, + "loss": 1.5932, + "theoretical_loss": 3.4674581629653227, + "tokens_seen": 1760952320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744040404040404, + "loss": 1.2436, + "theoretical_loss": 3.467447132460304, + "tokens_seen": 1761017856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743838383838384, + "loss": 1.2935, + "theoretical_loss": 3.467436102480711, + "tokens_seen": 1761083392 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047436363636363637, + "loss": 1.6615, + "theoretical_loss": 3.4674250730264986, + "tokens_seen": 1761148928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047434343434343433, + "loss": 1.4143, + "theoretical_loss": 3.4674140440976227, + "tokens_seen": 1761214464 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047432323232323234, + "loss": 1.5773, + "theoretical_loss": 3.467403015694039, + "tokens_seen": 1761280000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743030303030303, + "loss": 1.4823, + "theoretical_loss": 3.467391987815702, + "tokens_seen": 1761345536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742828282828283, + "loss": 1.282, + "theoretical_loss": 3.4673809604625676, + "tokens_seen": 1761411072 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047426262626262626, + "loss": 1.5621, + "theoretical_loss": 3.4673699336345916, + "tokens_seen": 1761476608 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.4690249264240265, + "objective/train/avg_token_score": 0.015736954286694527, + "objective/train/docs_used": 997243, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6199918985366821, + "objective/train/original_loss": 3.9470674991607666, + "objective/train/theoretical_loss": 3.4673589073317297, + "objective/train/tokens_used": 120401376, + "objective/train/unlikelihood_loss": 0.07392548024654388, + "theoretical_loss": 3.4673589073317297, + "tokens_seen": 1761542144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742424242424243, + "loss": 1.4835, + "theoretical_loss": 3.4673589073317297, + "tokens_seen": 1761542144 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047422222222222223, + "loss": 1.4607, + "theoretical_loss": 3.4673478815539363, + "tokens_seen": 1761607680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742020202020202, + "loss": 1.655, + "theoretical_loss": 3.4673368563011677, + "tokens_seen": 1761673216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741818181818182, + "loss": 1.4306, + "theoretical_loss": 3.467325831573379, + "tokens_seen": 1761738752 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047416161616161616, + "loss": 1.4634, + "theoretical_loss": 3.467314807370526, + "tokens_seen": 1761804288 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047414141414141417, + "loss": 1.4165, + "theoretical_loss": 3.4673037836925635, + "tokens_seen": 1761869824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741212121212121, + "loss": 1.4309, + "theoretical_loss": 3.467292760539448, + "tokens_seen": 1761935360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741010101010101, + "loss": 1.3544, + "theoretical_loss": 3.467281737911134, + "tokens_seen": 1762000896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004740808080808081, + "loss": 1.4597, + "theoretical_loss": 3.4672707158075777, + "tokens_seen": 1762066432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004740606060606061, + "loss": 1.5272, + "theoretical_loss": 3.467259694228734, + "tokens_seen": 1762131968 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047404040404040406, + "loss": 1.3645, + "theoretical_loss": 3.467248673174559, + "tokens_seen": 1762197504 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474020202020202, + "loss": 1.2698, + "theoretical_loss": 3.467237652645008, + "tokens_seen": 1762263040 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474, + "loss": 1.1518, + "theoretical_loss": 3.4672266326400365, + "tokens_seen": 1762328576 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473979797979798, + "loss": 1.446, + "theoretical_loss": 3.4672156131595995, + "tokens_seen": 1762394112 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473959595959596, + "loss": 1.4405, + "theoretical_loss": 3.4672045942036536, + "tokens_seen": 1762459648 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047393939393939395, + "loss": 1.2889, + "theoretical_loss": 3.4671935757721535, + "tokens_seen": 1762525184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739191919191919, + "loss": 1.389, + "theoretical_loss": 3.467182557865055, + "tokens_seen": 1762590720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047389898989898987, + "loss": 1.4165, + "theoretical_loss": 3.4671715404823136, + "tokens_seen": 1762656256 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047387878787878793, + "loss": 1.5323, + "theoretical_loss": 3.4671605236238854, + "tokens_seen": 1762721792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738585858585859, + "loss": 1.4547, + "theoretical_loss": 3.467149507289725, + "tokens_seen": 1762787328 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047383838383838385, + "loss": 1.3971, + "theoretical_loss": 3.4671384914797887, + "tokens_seen": 1762852864 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738181818181818, + "loss": 1.4061, + "theoretical_loss": 3.4671274761940314, + "tokens_seen": 1762918400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047379797979797976, + "loss": 1.2906, + "theoretical_loss": 3.4671164614324095, + "tokens_seen": 1762983936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737777777777778, + "loss": 1.4725, + "theoretical_loss": 3.4671054471948777, + "tokens_seen": 1763049472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737575757575758, + "loss": 1.2618, + "theoretical_loss": 3.467094433481392, + "tokens_seen": 1763115008 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.3298509418964386, + "objective/train/avg_token_score": 0.013608359731733799, + "objective/train/docs_used": 998400, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.140276312828064, + "objective/train/original_loss": 4.645390033721924, + "objective/train/theoretical_loss": 3.4670834202919085, + "objective/train/tokens_used": 122039776, + "objective/train/unlikelihood_loss": 0.06240946799516678, + "theoretical_loss": 3.4670834202919085, + "tokens_seen": 1763180544 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047373737373737374, + "loss": 1.3174, + "theoretical_loss": 3.4670834202919085, + "tokens_seen": 1763180544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737171717171717, + "loss": 1.6851, + "theoretical_loss": 3.4670724076263815, + "tokens_seen": 1763246080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736969696969697, + "loss": 1.3992, + "theoretical_loss": 3.4670613954847678, + "tokens_seen": 1763311616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736767676767677, + "loss": 1.482, + "theoretical_loss": 3.467050383867023, + "tokens_seen": 1763377152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736565656565657, + "loss": 1.4294, + "theoretical_loss": 3.467039372773102, + "tokens_seen": 1763442688 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047363636363636363, + "loss": 1.4823, + "theoretical_loss": 3.467028362202961, + "tokens_seen": 1763508224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736161616161616, + "loss": 1.4347, + "theoretical_loss": 3.467017352156555, + "tokens_seen": 1763573760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735959595959596, + "loss": 1.5324, + "theoretical_loss": 3.46700634263384, + "tokens_seen": 1763639296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735757575757576, + "loss": 1.2494, + "theoretical_loss": 3.466995333634772, + "tokens_seen": 1763704832 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047355555555555557, + "loss": 1.3624, + "theoretical_loss": 3.4669843251593058, + "tokens_seen": 1763770368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735353535353535, + "loss": 1.3935, + "theoretical_loss": 3.4669733172073975, + "tokens_seen": 1763835904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047351515151515154, + "loss": 1.4901, + "theoretical_loss": 3.466962309779003, + "tokens_seen": 1763901440 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734949494949495, + "loss": 1.3198, + "theoretical_loss": 3.4669513028740777, + "tokens_seen": 1763966976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734747474747475, + "loss": 1.2222, + "theoretical_loss": 3.466940296492577, + "tokens_seen": 1764032512 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047345454545454546, + "loss": 1.4412, + "theoretical_loss": 3.466929290634457, + "tokens_seen": 1764098048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734343434343434, + "loss": 1.2884, + "theoretical_loss": 3.466918285299673, + "tokens_seen": 1764163584 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047341414141414143, + "loss": 1.6086, + "theoretical_loss": 3.466907280488181, + "tokens_seen": 1764229120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733939393939394, + "loss": 1.3749, + "theoretical_loss": 3.4668962761999365, + "tokens_seen": 1764294656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733737373737374, + "loss": 1.2535, + "theoretical_loss": 3.4668852724348955, + "tokens_seen": 1764360192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047335353535353535, + "loss": 1.3333, + "theoretical_loss": 3.4668742691930126, + "tokens_seen": 1764425728 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047333333333333336, + "loss": 1.2971, + "theoretical_loss": 3.4668632664742454, + "tokens_seen": 1764491264 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733131313131313, + "loss": 1.3483, + "theoretical_loss": 3.4668522642785478, + "tokens_seen": 1764556800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732929292929293, + "loss": 1.2886, + "theoretical_loss": 3.466841262605876, + "tokens_seen": 1764622336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732727272727273, + "loss": 1.4083, + "theoretical_loss": 3.466830261456187, + "tokens_seen": 1764687872 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047325252525252525, + "loss": 1.4522, + "theoretical_loss": 3.4668192608294346, + "tokens_seen": 1764753408 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.30015885829925537, + "objective/train/avg_token_score": 0.025842411443591118, + "objective/train/docs_used": 998902, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9383648633956909, + "objective/train/original_loss": 4.241371154785156, + "objective/train/theoretical_loss": 3.4668082607255757, + "objective/train/tokens_used": 123678176, + "objective/train/unlikelihood_loss": 0.10302804410457611, + "theoretical_loss": 3.4668082607255757, + "tokens_seen": 1764818944 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047323232323232326, + "loss": 1.4569, + "theoretical_loss": 3.4668082607255757, + "tokens_seen": 1764818944 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732121212121212, + "loss": 1.4089, + "theoretical_loss": 3.4667972611445657, + "tokens_seen": 1764884480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731919191919192, + "loss": 1.4655, + "theoretical_loss": 3.46678626208636, + "tokens_seen": 1764950016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731717171717172, + "loss": 1.4081, + "theoretical_loss": 3.466775263550915, + "tokens_seen": 1765015552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731515151515152, + "loss": 1.5019, + "theoretical_loss": 3.466764265538186, + "tokens_seen": 1765081088 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047313131313131315, + "loss": 1.4618, + "theoretical_loss": 3.466753268048129, + "tokens_seen": 1765146624 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731111111111111, + "loss": 1.2713, + "theoretical_loss": 3.4667422710806997, + "tokens_seen": 1765212160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730909090909091, + "loss": 1.2932, + "theoretical_loss": 3.466731274635854, + "tokens_seen": 1765277696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730707070707071, + "loss": 1.6351, + "theoretical_loss": 3.466720278713547, + "tokens_seen": 1765343232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730505050505051, + "loss": 1.5274, + "theoretical_loss": 3.466709283313735, + "tokens_seen": 1765408768 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047303030303030304, + "loss": 1.4793, + "theoretical_loss": 3.4666982884363735, + "tokens_seen": 1765474304 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473010101010101, + "loss": 1.4963, + "theoretical_loss": 3.466687294081419, + "tokens_seen": 1765539840 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472989898989899, + "loss": 1.3953, + "theoretical_loss": 3.466676300248827, + "tokens_seen": 1765605376 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472969696969697, + "loss": 1.3848, + "theoretical_loss": 3.4666653069385527, + "tokens_seen": 1765670912 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472949494949495, + "loss": 1.3043, + "theoretical_loss": 3.466654314150552, + "tokens_seen": 1765736448 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047292929292929293, + "loss": 1.3659, + "theoretical_loss": 3.4666433218847814, + "tokens_seen": 1765801984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729090909090909, + "loss": 1.2315, + "theoretical_loss": 3.466632330141196, + "tokens_seen": 1765867520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728888888888889, + "loss": 1.412, + "theoretical_loss": 3.4666213389197518, + "tokens_seen": 1765933056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728686868686869, + "loss": 1.527, + "theoretical_loss": 3.4666103482204056, + "tokens_seen": 1765998592 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047284848484848487, + "loss": 1.1406, + "theoretical_loss": 3.466599358043111, + "tokens_seen": 1766064128 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047282828282828283, + "loss": 1.4347, + "theoretical_loss": 3.4665883683878262, + "tokens_seen": 1766129664 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728080808080808, + "loss": 1.2191, + "theoretical_loss": 3.466577379254506, + "tokens_seen": 1766195200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727878787878788, + "loss": 1.3334, + "theoretical_loss": 3.4665663906431057, + "tokens_seen": 1766260736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727676767676768, + "loss": 1.3408, + "theoretical_loss": 3.466555402553582, + "tokens_seen": 1766326272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047274747474747476, + "loss": 1.424, + "theoretical_loss": 3.466544414985891, + "tokens_seen": 1766391808 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.5058650970458984, + "objective/train/avg_token_score": 0.020609397441148758, + "objective/train/docs_used": 999561, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6941487789154053, + "objective/train/original_loss": 3.954047918319702, + "objective/train/theoretical_loss": 3.4665334279399875, + "objective/train/tokens_used": 125316576, + "objective/train/unlikelihood_loss": 0.04431372508406639, + "theoretical_loss": 3.4665334279399875, + "tokens_seen": 1766457344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727272727272727, + "loss": 1.5081, + "theoretical_loss": 3.4665334279399875, + "tokens_seen": 1766457344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727070707070707, + "loss": 1.2911, + "theoretical_loss": 3.4665224414158278, + "tokens_seen": 1766522880 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047268686868686874, + "loss": 1.4671, + "theoretical_loss": 3.466511455413368, + "tokens_seen": 1766588416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726666666666667, + "loss": 1.3627, + "theoretical_loss": 3.466500469932564, + "tokens_seen": 1766653952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047264646464646466, + "loss": 1.6863, + "theoretical_loss": 3.466489484973371, + "tokens_seen": 1766719488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726262626262626, + "loss": 1.3115, + "theoretical_loss": 3.4664785005357457, + "tokens_seen": 1766785024 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047260606060606057, + "loss": 1.4088, + "theoretical_loss": 3.4664675166196437, + "tokens_seen": 1766850560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047258585858585864, + "loss": 1.2904, + "theoretical_loss": 3.4664565332250215, + "tokens_seen": 1766916096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725656565656566, + "loss": 1.5834, + "theoretical_loss": 3.4664455503518337, + "tokens_seen": 1766981632 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047254545454545455, + "loss": 1.5353, + "theoretical_loss": 3.466434568000037, + "tokens_seen": 1767047168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725252525252525, + "loss": 1.2129, + "theoretical_loss": 3.4664235861695873, + "tokens_seen": 1767112704 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725050505050505, + "loss": 1.4923, + "theoretical_loss": 3.46641260486044, + "tokens_seen": 1767178240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047248484848484853, + "loss": 1.6744, + "theoretical_loss": 3.466401624072552, + "tokens_seen": 1767243776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724646464646465, + "loss": 1.1128, + "theoretical_loss": 3.4663906438058785, + "tokens_seen": 1767309312 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047244444444444444, + "loss": 1.5299, + "theoretical_loss": 3.4663796640603755, + "tokens_seen": 1767374848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724242424242424, + "loss": 1.4135, + "theoretical_loss": 3.466368684835999, + "tokens_seen": 1767440384 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724040404040404, + "loss": 1.3394, + "theoretical_loss": 3.466357706132705, + "tokens_seen": 1767505920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723838383838384, + "loss": 1.3015, + "theoretical_loss": 3.46634672795045, + "tokens_seen": 1767571456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723636363636364, + "loss": 1.3472, + "theoretical_loss": 3.466335750289189, + "tokens_seen": 1767636992 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047234343434343433, + "loss": 1.2866, + "theoretical_loss": 3.466324773148878, + "tokens_seen": 1767702528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047232323232323235, + "loss": 1.6653, + "theoretical_loss": 3.466313796529474, + "tokens_seen": 1767768064 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723030303030303, + "loss": 1.5335, + "theoretical_loss": 3.466302820430932, + "tokens_seen": 1767833600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004722828282828283, + "loss": 1.5068, + "theoretical_loss": 3.466291844853208, + "tokens_seen": 1767899136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047226262626262627, + "loss": 1.3592, + "theoretical_loss": 3.4662808697962584, + "tokens_seen": 1767964672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004722424242424242, + "loss": 1.4486, + "theoretical_loss": 3.466269895260039, + "tokens_seen": 1768030208 + }, + { + "epoch": 0.06, + "objective/train/aligned_token_count": 0.3491568863391876, + "objective/train/avg_token_score": 0.024049675092101097, + "objective/train/docs_used": 1000589, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.332099437713623, + "objective/train/original_loss": 4.583323001861572, + "objective/train/theoretical_loss": 3.466258921244506, + "objective/train/tokens_used": 126954976, + "objective/train/unlikelihood_loss": 0.07067796587944031, + "theoretical_loss": 3.466258921244506, + "tokens_seen": 1768095744 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047222222222222224, + "loss": 1.5076, + "theoretical_loss": 3.466258921244506, + "tokens_seen": 1768095744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004722020202020202, + "loss": 1.2966, + "theoretical_loss": 3.466247947749615, + "tokens_seen": 1768161280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721818181818182, + "loss": 1.2774, + "theoretical_loss": 3.466236974775322, + "tokens_seen": 1768226816 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047216161616161616, + "loss": 1.267, + "theoretical_loss": 3.4662260023215836, + "tokens_seen": 1768292352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721414141414142, + "loss": 1.362, + "theoretical_loss": 3.466215030388356, + "tokens_seen": 1768357888 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047212121212121213, + "loss": 1.427, + "theoretical_loss": 3.4662040589755936, + "tokens_seen": 1768423424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721010101010101, + "loss": 1.2644, + "theoretical_loss": 3.466193088083254, + "tokens_seen": 1768488960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720808080808081, + "loss": 1.5104, + "theoretical_loss": 3.466182117711293, + "tokens_seen": 1768554496 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047206060606060606, + "loss": 1.1877, + "theoretical_loss": 3.466171147859666, + "tokens_seen": 1768620032 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047204040404040407, + "loss": 1.3425, + "theoretical_loss": 3.46616017852833, + "tokens_seen": 1768685568 + }, + { + "epoch": 0.07, + "learning_rate": 0.000472020202020202, + "loss": 1.1985, + "theoretical_loss": 3.4661492097172397, + "tokens_seen": 1768751104 + }, + { + "epoch": 0.07, + "learning_rate": 0.000472, + "loss": 1.4138, + "theoretical_loss": 3.466138241426352, + "tokens_seen": 1768816640 + }, + { + "epoch": 0.07, + "learning_rate": 0.000471979797979798, + "loss": 1.3324, + "theoretical_loss": 3.466127273655623, + "tokens_seen": 1768882176 + }, + { + "epoch": 0.07, + "learning_rate": 0.000471959595959596, + "loss": 1.4765, + "theoretical_loss": 3.466116306405009, + "tokens_seen": 1768947712 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047193939393939396, + "loss": 1.2163, + "theoretical_loss": 3.466105339674465, + "tokens_seen": 1769013248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719191919191919, + "loss": 1.319, + "theoretical_loss": 3.466094373463948, + "tokens_seen": 1769078784 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718989898989899, + "loss": 1.3871, + "theoretical_loss": 3.4660834077734144, + "tokens_seen": 1769144320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718787878787879, + "loss": 1.3804, + "theoretical_loss": 3.466072442602819, + "tokens_seen": 1769209856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718585858585859, + "loss": 1.3414, + "theoretical_loss": 3.4660614779521186, + "tokens_seen": 1769275392 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047183838383838385, + "loss": 1.391, + "theoretical_loss": 3.4660505138212696, + "tokens_seen": 1769340928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718181818181818, + "loss": 1.4082, + "theoretical_loss": 3.466039550210228, + "tokens_seen": 1769406464 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047179797979797977, + "loss": 1.4775, + "theoretical_loss": 3.4660285871189496, + "tokens_seen": 1769472000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047177777777777783, + "loss": 1.4404, + "theoretical_loss": 3.4660176245473906, + "tokens_seen": 1769537536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717575757575758, + "loss": 1.4915, + "theoretical_loss": 3.4660066624955066, + "tokens_seen": 1769603072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047173737373737374, + "loss": 1.7011, + "theoretical_loss": 3.465995700963255, + "tokens_seen": 1769668608 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.48918622732162476, + "objective/train/avg_token_score": 0.019492249935865402, + "objective/train/docs_used": 1001288, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7274926900863647, + "objective/train/original_loss": 3.8678648471832275, + "objective/train/theoretical_loss": 3.4659847399505903, + "objective/train/tokens_used": 128593376, + "objective/train/unlikelihood_loss": 0.081143319606781, + "theoretical_loss": 3.4659847399505903, + "tokens_seen": 1769734144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717171717171717, + "loss": 1.3668, + "theoretical_loss": 3.4659847399505903, + "tokens_seen": 1769734144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047169696969696966, + "loss": 1.3583, + "theoretical_loss": 3.46597377945747, + "tokens_seen": 1769799680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716767676767677, + "loss": 1.3333, + "theoretical_loss": 3.46596281948385, + "tokens_seen": 1769865216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716565656565657, + "loss": 1.3922, + "theoretical_loss": 3.465951860029686, + "tokens_seen": 1769930752 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047163636363636364, + "loss": 1.519, + "theoretical_loss": 3.4659409010949345, + "tokens_seen": 1769996288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716161616161616, + "loss": 1.4696, + "theoretical_loss": 3.4659299426795513, + "tokens_seen": 1770061824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715959595959596, + "loss": 1.3336, + "theoretical_loss": 3.4659189847834924, + "tokens_seen": 1770127360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715757575757576, + "loss": 1.5364, + "theoretical_loss": 3.465908027406715, + "tokens_seen": 1770192896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715555555555556, + "loss": 1.3424, + "theoretical_loss": 3.465897070549174, + "tokens_seen": 1770258432 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047153535353535353, + "loss": 1.3519, + "theoretical_loss": 3.4658861142108264, + "tokens_seen": 1770323968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715151515151515, + "loss": 1.1967, + "theoretical_loss": 3.4658751583916283, + "tokens_seen": 1770389504 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047149494949494955, + "loss": 1.2003, + "theoretical_loss": 3.4658642030915354, + "tokens_seen": 1770455040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714747474747475, + "loss": 1.1804, + "theoretical_loss": 3.4658532483105042, + "tokens_seen": 1770520576 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047145454545454547, + "loss": 1.4514, + "theoretical_loss": 3.4658422940484908, + "tokens_seen": 1770586112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714343434343434, + "loss": 1.2616, + "theoretical_loss": 3.465831340305452, + "tokens_seen": 1770651648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714141414141414, + "loss": 1.1625, + "theoretical_loss": 3.4658203870813433, + "tokens_seen": 1770717184 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047139393939393944, + "loss": 1.7548, + "theoretical_loss": 3.465809434376121, + "tokens_seen": 1770782720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713737373737374, + "loss": 1.2719, + "theoretical_loss": 3.465798482189742, + "tokens_seen": 1770848256 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047135353535353536, + "loss": 1.3652, + "theoretical_loss": 3.465787530522161, + "tokens_seen": 1770913792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713333333333333, + "loss": 1.4807, + "theoretical_loss": 3.4657765793733355, + "tokens_seen": 1770979328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713131313131313, + "loss": 1.4566, + "theoretical_loss": 3.4657656287432212, + "tokens_seen": 1771044864 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047129292929292934, + "loss": 1.31, + "theoretical_loss": 3.465754678631775, + "tokens_seen": 1771110400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712727272727273, + "loss": 1.6366, + "theoretical_loss": 3.4657437290389526, + "tokens_seen": 1771175936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047125252525252525, + "loss": 1.1361, + "theoretical_loss": 3.4657327799647106, + "tokens_seen": 1771241472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712323232323232, + "loss": 1.3109, + "theoretical_loss": 3.4657218314090046, + "tokens_seen": 1771307008 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.5297531485557556, + "objective/train/avg_token_score": 0.0018574605928733945, + "objective/train/docs_used": 1002447, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.670203447341919, + "objective/train/original_loss": 3.6806411743164062, + "objective/train/theoretical_loss": 3.4657108833717913, + "objective/train/tokens_used": 130231776, + "objective/train/unlikelihood_loss": 0.10642728209495544, + "theoretical_loss": 3.4657108833717913, + "tokens_seen": 1771372544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712121212121212, + "loss": 1.7502, + "theoretical_loss": 3.4657108833717913, + "tokens_seen": 1771372544 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047119191919191923, + "loss": 1.4412, + "theoretical_loss": 3.465699935853027, + "tokens_seen": 1771438080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711717171717172, + "loss": 1.2746, + "theoretical_loss": 3.465688988852668, + "tokens_seen": 1771503616 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047115151515151514, + "loss": 1.5396, + "theoretical_loss": 3.46567804237067, + "tokens_seen": 1771569152 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047113131313131316, + "loss": 1.2664, + "theoretical_loss": 3.4656670964069907, + "tokens_seen": 1771634688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711111111111111, + "loss": 1.6049, + "theoretical_loss": 3.4656561509615846, + "tokens_seen": 1771700224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710909090909091, + "loss": 1.5435, + "theoretical_loss": 3.4656452060344085, + "tokens_seen": 1771765760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710707070707071, + "loss": 1.2803, + "theoretical_loss": 3.4656342616254197, + "tokens_seen": 1771831296 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047105050505050504, + "loss": 1.6564, + "theoretical_loss": 3.4656233177345737, + "tokens_seen": 1771896832 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047103030303030305, + "loss": 1.6557, + "theoretical_loss": 3.465612374361827, + "tokens_seen": 1771962368 + }, + { + "epoch": 0.07, + "learning_rate": 0.000471010101010101, + "loss": 1.2512, + "theoretical_loss": 3.4656014315071357, + "tokens_seen": 1772027904 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470989898989899, + "loss": 1.3483, + "theoretical_loss": 3.4655904891704563, + "tokens_seen": 1772093440 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047096969696969697, + "loss": 1.4258, + "theoretical_loss": 3.4655795473517452, + "tokens_seen": 1772158976 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470949494949495, + "loss": 1.414, + "theoretical_loss": 3.465568606050958, + "tokens_seen": 1772224512 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047092929292929294, + "loss": 1.2494, + "theoretical_loss": 3.465557665268052, + "tokens_seen": 1772290048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709090909090909, + "loss": 1.2409, + "theoretical_loss": 3.4655467250029837, + "tokens_seen": 1772355584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708888888888889, + "loss": 1.2723, + "theoretical_loss": 3.4655357852557085, + "tokens_seen": 1772421120 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047086868686868687, + "loss": 1.353, + "theoretical_loss": 3.4655248460261827, + "tokens_seen": 1772486656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708484848484849, + "loss": 1.7172, + "theoretical_loss": 3.4655139073143637, + "tokens_seen": 1772552192 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047082828282828283, + "loss": 1.4032, + "theoretical_loss": 3.465502969120207, + "tokens_seen": 1772617728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708080808080808, + "loss": 1.4746, + "theoretical_loss": 3.4654920314436697, + "tokens_seen": 1772683264 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707878787878788, + "loss": 1.3164, + "theoretical_loss": 3.465481094284707, + "tokens_seen": 1772748800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707676767676768, + "loss": 1.6484, + "theoretical_loss": 3.465470157643277, + "tokens_seen": 1772814336 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047074747474747477, + "loss": 1.4666, + "theoretical_loss": 3.4654592215193345, + "tokens_seen": 1772879872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707272727272727, + "loss": 1.616, + "theoretical_loss": 3.465448285912836, + "tokens_seen": 1772945408 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.42247068881988525, + "objective/train/avg_token_score": 0.04283915087580681, + "objective/train/docs_used": 1002969, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3265541791915894, + "objective/train/original_loss": 4.510785102844238, + "objective/train/theoretical_loss": 3.465437350823739, + "objective/train/tokens_used": 131870176, + "objective/train/unlikelihood_loss": 0.05501260235905647, + "theoretical_loss": 3.465437350823739, + "tokens_seen": 1773010944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707070707070707, + "loss": 1.5543, + "theoretical_loss": 3.465437350823739, + "tokens_seen": 1773010944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706868686868687, + "loss": 1.4789, + "theoretical_loss": 3.465426416251999, + "tokens_seen": 1773076480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706666666666667, + "loss": 1.3916, + "theoretical_loss": 3.4654154821975727, + "tokens_seen": 1773142016 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047064646464646466, + "loss": 1.5454, + "theoretical_loss": 3.465404548660416, + "tokens_seen": 1773207552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706262626262626, + "loss": 1.4936, + "theoretical_loss": 3.4653936156404868, + "tokens_seen": 1773273088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706060606060606, + "loss": 1.6112, + "theoretical_loss": 3.4653826831377397, + "tokens_seen": 1773338624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047058585858585864, + "loss": 1.4224, + "theoretical_loss": 3.465371751152132, + "tokens_seen": 1773404160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705656565656566, + "loss": 1.3528, + "theoretical_loss": 3.46536081968362, + "tokens_seen": 1773469696 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047054545454545455, + "loss": 1.5744, + "theoretical_loss": 3.4653498887321605, + "tokens_seen": 1773535232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705252525252525, + "loss": 1.6217, + "theoretical_loss": 3.465338958297709, + "tokens_seen": 1773600768 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047050505050505047, + "loss": 1.3906, + "theoretical_loss": 3.465328028380223, + "tokens_seen": 1773666304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047048484848484853, + "loss": 1.5699, + "theoretical_loss": 3.465317098979659, + "tokens_seen": 1773731840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704646464646465, + "loss": 1.4396, + "theoretical_loss": 3.465306170095972, + "tokens_seen": 1773797376 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047044444444444445, + "loss": 1.2902, + "theoretical_loss": 3.46529524172912, + "tokens_seen": 1773862912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704242424242424, + "loss": 1.526, + "theoretical_loss": 3.4652843138790583, + "tokens_seen": 1773928448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704040404040404, + "loss": 1.373, + "theoretical_loss": 3.4652733865457446, + "tokens_seen": 1773993984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703838383838384, + "loss": 1.4229, + "theoretical_loss": 3.465262459729135, + "tokens_seen": 1774059520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703636363636364, + "loss": 1.3665, + "theoretical_loss": 3.4652515334291847, + "tokens_seen": 1774125056 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047034343434343434, + "loss": 1.4426, + "theoretical_loss": 3.4652406076458515, + "tokens_seen": 1774190592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703232323232323, + "loss": 1.308, + "theoretical_loss": 3.465229682379092, + "tokens_seen": 1774256128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703030303030303, + "loss": 1.2964, + "theoretical_loss": 3.465218757628862, + "tokens_seen": 1774321664 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702828282828283, + "loss": 1.3539, + "theoretical_loss": 3.4652078333951186, + "tokens_seen": 1774387200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702626262626263, + "loss": 1.4111, + "theoretical_loss": 3.4651969096778177, + "tokens_seen": 1774452736 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047024242424242423, + "loss": 1.2421, + "theoretical_loss": 3.465185986476916, + "tokens_seen": 1774518272 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047022222222222224, + "loss": 1.3518, + "theoretical_loss": 3.465175063792371, + "tokens_seen": 1774583808 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.2966764271259308, + "objective/train/avg_token_score": 0.02469574473798275, + "objective/train/docs_used": 1004180, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0019471645355225, + "objective/train/original_loss": 4.505530834197998, + "objective/train/theoretical_loss": 3.465164141624137, + "objective/train/tokens_used": 133508576, + "objective/train/unlikelihood_loss": 0.06781195849180222, + "theoretical_loss": 3.465164141624137, + "tokens_seen": 1774649344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702020202020202, + "loss": 1.4618, + "theoretical_loss": 3.465164141624137, + "tokens_seen": 1774649344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701818181818182, + "loss": 1.6281, + "theoretical_loss": 3.465153219972173, + "tokens_seen": 1774714880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047016161616161617, + "loss": 1.3271, + "theoretical_loss": 3.4651422988364335, + "tokens_seen": 1774780416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701414141414141, + "loss": 1.3957, + "theoretical_loss": 3.4651313782168764, + "tokens_seen": 1774845952 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047012121212121214, + "loss": 1.5541, + "theoretical_loss": 3.465120458113458, + "tokens_seen": 1774911488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701010101010101, + "loss": 1.435, + "theoretical_loss": 3.4651095385261343, + "tokens_seen": 1774977024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700808080808081, + "loss": 1.6075, + "theoretical_loss": 3.4650986194548623, + "tokens_seen": 1775042560 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047006060606060606, + "loss": 1.3927, + "theoretical_loss": 3.4650877008995984, + "tokens_seen": 1775108096 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047004040404040407, + "loss": 1.2209, + "theoretical_loss": 3.4650767828602995, + "tokens_seen": 1775173632 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047002020202020203, + "loss": 1.3741, + "theoretical_loss": 3.4650658653369213, + "tokens_seen": 1775239168 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047, + "loss": 1.46, + "theoretical_loss": 3.4650549483294215, + "tokens_seen": 1775304704 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469979797979798, + "loss": 1.4385, + "theoretical_loss": 3.4650440318377562, + "tokens_seen": 1775370240 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046995959595959595, + "loss": 1.3606, + "theoretical_loss": 3.465033115861882, + "tokens_seen": 1775435776 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046993939393939396, + "loss": 1.1657, + "theoretical_loss": 3.465022200401755, + "tokens_seen": 1775501312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699191919191919, + "loss": 1.6816, + "theoretical_loss": 3.465011285457332, + "tokens_seen": 1775566848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698989898989899, + "loss": 1.3001, + "theoretical_loss": 3.4650003710285704, + "tokens_seen": 1775632384 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698787878787879, + "loss": 1.1577, + "theoretical_loss": 3.4649894571154265, + "tokens_seen": 1775697920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698585858585859, + "loss": 1.3568, + "theoretical_loss": 3.464978543717856, + "tokens_seen": 1775763456 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046983838383838386, + "loss": 1.614, + "theoretical_loss": 3.464967630835816, + "tokens_seen": 1775828992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698181818181818, + "loss": 1.2982, + "theoretical_loss": 3.464956718469264, + "tokens_seen": 1775894528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697979797979798, + "loss": 1.5625, + "theoretical_loss": 3.4649458066181555, + "tokens_seen": 1775960064 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697777777777778, + "loss": 1.6927, + "theoretical_loss": 3.4649348952824477, + "tokens_seen": 1776025600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697575757575758, + "loss": 1.4541, + "theoretical_loss": 3.4649239844620974, + "tokens_seen": 1776091136 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046973737373737375, + "loss": 1.4913, + "theoretical_loss": 3.46491307415706, + "tokens_seen": 1776156672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697171717171717, + "loss": 1.5123, + "theoretical_loss": 3.464902164367294, + "tokens_seen": 1776222208 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.2969208061695099, + "objective/train/avg_token_score": 0.01136105228215456, + "objective/train/docs_used": 1004852, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8510345816612244, + "objective/train/original_loss": 3.909015417098999, + "objective/train/theoretical_loss": 3.4648912550927546, + "objective/train/tokens_used": 135146976, + "objective/train/unlikelihood_loss": 0.0887172669172287, + "theoretical_loss": 3.4648912550927546, + "tokens_seen": 1776287744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696969696969697, + "loss": 1.363, + "theoretical_loss": 3.4648912550927546, + "tokens_seen": 1776287744 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046967676767676773, + "loss": 1.4334, + "theoretical_loss": 3.4648803463333993, + "tokens_seen": 1776353280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696565656565657, + "loss": 1.4213, + "theoretical_loss": 3.4648694380891847, + "tokens_seen": 1776418816 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046963636363636364, + "loss": 1.2654, + "theoretical_loss": 3.4648585303600665, + "tokens_seen": 1776484352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696161616161616, + "loss": 1.3323, + "theoretical_loss": 3.464847623146003, + "tokens_seen": 1776549888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695959595959596, + "loss": 1.4284, + "theoretical_loss": 3.464836716446949, + "tokens_seen": 1776615424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695757575757576, + "loss": 1.2791, + "theoretical_loss": 3.464825810262863, + "tokens_seen": 1776680960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695555555555556, + "loss": 1.1452, + "theoretical_loss": 3.4648149045937005, + "tokens_seen": 1776746496 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046953535353535354, + "loss": 1.5327, + "theoretical_loss": 3.464803999439418, + "tokens_seen": 1776812032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695151515151515, + "loss": 1.4731, + "theoretical_loss": 3.4647930947999743, + "tokens_seen": 1776877568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694949494949495, + "loss": 1.4519, + "theoretical_loss": 3.464782190675323, + "tokens_seen": 1776943104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694747474747475, + "loss": 1.4544, + "theoretical_loss": 3.4647712870654237, + "tokens_seen": 1777008640 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046945454545454547, + "loss": 1.46, + "theoretical_loss": 3.464760383970231, + "tokens_seen": 1777074176 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046943434343434343, + "loss": 1.3681, + "theoretical_loss": 3.4647494813897026, + "tokens_seen": 1777139712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694141414141414, + "loss": 1.4295, + "theoretical_loss": 3.464738579323795, + "tokens_seen": 1777205248 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046939393939393945, + "loss": 1.5011, + "theoretical_loss": 3.4647276777724647, + "tokens_seen": 1777270784 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693737373737374, + "loss": 1.3782, + "theoretical_loss": 3.4647167767356692, + "tokens_seen": 1777336320 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046935353535353536, + "loss": 1.5472, + "theoretical_loss": 3.4647058762133645, + "tokens_seen": 1777401856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693333333333333, + "loss": 1.2642, + "theoretical_loss": 3.4646949762055077, + "tokens_seen": 1777467392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693131313131313, + "loss": 1.292, + "theoretical_loss": 3.4646840767120555, + "tokens_seen": 1777532928 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046929292929292934, + "loss": 1.5226, + "theoretical_loss": 3.4646731777329647, + "tokens_seen": 1777598464 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692727272727273, + "loss": 1.4523, + "theoretical_loss": 3.4646622792681923, + "tokens_seen": 1777664000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046925252525252526, + "loss": 1.3278, + "theoretical_loss": 3.4646513813176942, + "tokens_seen": 1777729536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692323232323232, + "loss": 1.6332, + "theoretical_loss": 3.464640483881428, + "tokens_seen": 1777795072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692121212121212, + "loss": 1.388, + "theoretical_loss": 3.46462958695935, + "tokens_seen": 1777860608 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.34634652733802795, + "objective/train/avg_token_score": 0.013553972356021404, + "objective/train/docs_used": 1006253, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2541776895523071, + "objective/train/original_loss": 5.052961826324463, + "objective/train/theoretical_loss": 3.4646186905514176, + "objective/train/tokens_used": 136785376, + "objective/train/unlikelihood_loss": 0.05251467227935791, + "theoretical_loss": 3.4646186905514176, + "tokens_seen": 1777926144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046919191919191924, + "loss": 1.3604, + "theoretical_loss": 3.4646186905514176, + "tokens_seen": 1777926144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691717171717172, + "loss": 1.3728, + "theoretical_loss": 3.464607794657587, + "tokens_seen": 1777991680 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046915151515151515, + "loss": 1.4799, + "theoretical_loss": 3.464596899277815, + "tokens_seen": 1778057216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691313131313131, + "loss": 1.4162, + "theoretical_loss": 3.464586004412059, + "tokens_seen": 1778122752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691111111111111, + "loss": 1.364, + "theoretical_loss": 3.464575110060275, + "tokens_seen": 1778188288 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046909090909090913, + "loss": 1.2986, + "theoretical_loss": 3.4645642162224206, + "tokens_seen": 1778253824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690707070707071, + "loss": 1.3651, + "theoretical_loss": 3.4645533228984515, + "tokens_seen": 1778319360 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046905050505050504, + "loss": 1.2544, + "theoretical_loss": 3.464542430088326, + "tokens_seen": 1778384896 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046903030303030305, + "loss": 1.3842, + "theoretical_loss": 3.4645315377920003, + "tokens_seen": 1778450432 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469010101010101, + "loss": 1.4251, + "theoretical_loss": 3.46452064600943, + "tokens_seen": 1778515968 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468989898989899, + "loss": 1.5926, + "theoretical_loss": 3.4645097547405745, + "tokens_seen": 1778581504 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468969696969697, + "loss": 1.4541, + "theoretical_loss": 3.4644988639853884, + "tokens_seen": 1778647040 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046894949494949494, + "loss": 1.2371, + "theoretical_loss": 3.4644879737438297, + "tokens_seen": 1778712576 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046892929292929295, + "loss": 1.4936, + "theoretical_loss": 3.4644770840158543, + "tokens_seen": 1778778112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689090909090909, + "loss": 1.3229, + "theoretical_loss": 3.46446619480142, + "tokens_seen": 1778843648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688888888888889, + "loss": 1.2463, + "theoretical_loss": 3.4644553061004832, + "tokens_seen": 1778909184 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046886868686868687, + "loss": 1.4809, + "theoretical_loss": 3.4644444179130014, + "tokens_seen": 1778974720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688484848484849, + "loss": 1.2319, + "theoretical_loss": 3.46443353023893, + "tokens_seen": 1779040256 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046882828282828284, + "loss": 1.4577, + "theoretical_loss": 3.464422643078228, + "tokens_seen": 1779105792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688080808080808, + "loss": 1.4697, + "theoretical_loss": 3.4644117564308505, + "tokens_seen": 1779171328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687878787878788, + "loss": 1.4904, + "theoretical_loss": 3.464400870296755, + "tokens_seen": 1779236864 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046876767676767676, + "loss": 1.5906, + "theoretical_loss": 3.4643899846758983, + "tokens_seen": 1779302400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687474747474748, + "loss": 1.3874, + "theoretical_loss": 3.4643790995682373, + "tokens_seen": 1779367936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046872727272727273, + "loss": 1.6713, + "theoretical_loss": 3.46436821497373, + "tokens_seen": 1779433472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687070707070707, + "loss": 1.4633, + "theoretical_loss": 3.464357330892331, + "tokens_seen": 1779499008 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.4292522072792053, + "objective/train/avg_token_score": 0.015219959430396557, + "objective/train/docs_used": 1006973, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.337504506111145, + "objective/train/original_loss": 4.010739326477051, + "objective/train/theoretical_loss": 3.464346447323999, + "objective/train/tokens_used": 138423776, + "objective/train/unlikelihood_loss": 0.09005622565746307, + "theoretical_loss": 3.464346447323999, + "tokens_seen": 1779564544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686868686868687, + "loss": 1.3108, + "theoretical_loss": 3.464346447323999, + "tokens_seen": 1779564544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686666666666667, + "loss": 1.5185, + "theoretical_loss": 3.4643355642686906, + "tokens_seen": 1779630080 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046864646464646467, + "loss": 1.4462, + "theoretical_loss": 3.464324681726363, + "tokens_seen": 1779695616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686262626262626, + "loss": 1.2584, + "theoretical_loss": 3.4643137996969724, + "tokens_seen": 1779761152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686060606060606, + "loss": 1.5357, + "theoretical_loss": 3.4643029181804756, + "tokens_seen": 1779826688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685858585858586, + "loss": 1.4705, + "theoretical_loss": 3.464292037176831, + "tokens_seen": 1779892224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685656565656566, + "loss": 1.2145, + "theoretical_loss": 3.4642811566859937, + "tokens_seen": 1779957760 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046854545454545456, + "loss": 1.4919, + "theoretical_loss": 3.4642702767079214, + "tokens_seen": 1780023296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685252525252525, + "loss": 1.4393, + "theoretical_loss": 3.4642593972425715, + "tokens_seen": 1780088832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685050505050505, + "loss": 1.3427, + "theoretical_loss": 3.4642485182899008, + "tokens_seen": 1780154368 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046848484848484854, + "loss": 1.6959, + "theoretical_loss": 3.464237639849866, + "tokens_seen": 1780219904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684646464646465, + "loss": 1.2308, + "theoretical_loss": 3.464226761922424, + "tokens_seen": 1780285440 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046844444444444445, + "loss": 1.2129, + "theoretical_loss": 3.464215884507532, + "tokens_seen": 1780350976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684242424242424, + "loss": 1.4428, + "theoretical_loss": 3.464205007605147, + "tokens_seen": 1780416512 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046840404040404037, + "loss": 1.579, + "theoretical_loss": 3.464194131215226, + "tokens_seen": 1780482048 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046838383838383843, + "loss": 1.3781, + "theoretical_loss": 3.4641832553377254, + "tokens_seen": 1780547584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683636363636364, + "loss": 1.3164, + "theoretical_loss": 3.464172379972603, + "tokens_seen": 1780613120 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046834343434343435, + "loss": 1.5964, + "theoretical_loss": 3.4641615051198156, + "tokens_seen": 1780678656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683232323232323, + "loss": 1.4625, + "theoretical_loss": 3.4641506307793204, + "tokens_seen": 1780744192 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046830303030303026, + "loss": 1.3468, + "theoretical_loss": 3.4641397569510737, + "tokens_seen": 1780809728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682828282828283, + "loss": 1.4519, + "theoretical_loss": 3.464128883635033, + "tokens_seen": 1780875264 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682626262626263, + "loss": 1.3987, + "theoretical_loss": 3.4641180108311556, + "tokens_seen": 1780940800 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046824242424242424, + "loss": 1.3296, + "theoretical_loss": 3.4641071385393976, + "tokens_seen": 1781006336 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682222222222222, + "loss": 1.5257, + "theoretical_loss": 3.464096266759717, + "tokens_seen": 1781071872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682020202020202, + "loss": 1.5309, + "theoretical_loss": 3.4640853954920705, + "tokens_seen": 1781137408 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.3690738081932068, + "objective/train/avg_token_score": 0.019216934219002724, + "objective/train/docs_used": 1008497, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3794710636138916, + "objective/train/original_loss": 4.352183818817139, + "objective/train/theoretical_loss": 3.4640745247364153, + "objective/train/tokens_used": 140062176, + "objective/train/unlikelihood_loss": 0.05833863466978073, + "theoretical_loss": 3.4640745247364153, + "tokens_seen": 1781202944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681818181818182, + "loss": 1.5147, + "theoretical_loss": 3.4640745247364153, + "tokens_seen": 1781202944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681616161616162, + "loss": 1.2851, + "theoretical_loss": 3.4640636544927075, + "tokens_seen": 1781268480 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046814141414141413, + "loss": 1.5317, + "theoretical_loss": 3.4640527847609057, + "tokens_seen": 1781334016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681212121212121, + "loss": 1.3507, + "theoretical_loss": 3.4640419155409665, + "tokens_seen": 1781399552 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046810101010101015, + "loss": 1.3736, + "theoretical_loss": 3.4640310468328455, + "tokens_seen": 1781465088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680808080808081, + "loss": 1.4927, + "theoretical_loss": 3.4640201786365017, + "tokens_seen": 1781530624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046806060606060607, + "loss": 1.3966, + "theoretical_loss": 3.4640093109518917, + "tokens_seen": 1781596160 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468040404040404, + "loss": 1.316, + "theoretical_loss": 3.463998443778972, + "tokens_seen": 1781661696 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046802020202020203, + "loss": 1.2633, + "theoretical_loss": 3.4639875771177, + "tokens_seen": 1781727232 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046800000000000005, + "loss": 1.2474, + "theoretical_loss": 3.4639767109680326, + "tokens_seen": 1781792768 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467979797979798, + "loss": 1.5079, + "theoretical_loss": 3.463965845329927, + "tokens_seen": 1781858304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046795959595959596, + "loss": 1.3738, + "theoretical_loss": 3.46395498020334, + "tokens_seen": 1781923840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679393939393939, + "loss": 1.4382, + "theoretical_loss": 3.4639441155882302, + "tokens_seen": 1781989376 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046791919191919193, + "loss": 1.4504, + "theoretical_loss": 3.463933251484553, + "tokens_seen": 1782054912 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046789898989898994, + "loss": 1.5389, + "theoretical_loss": 3.463922387892266, + "tokens_seen": 1782120448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678787878787879, + "loss": 1.3819, + "theoretical_loss": 3.4639115248113264, + "tokens_seen": 1782185984 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046785858585858585, + "loss": 1.3085, + "theoretical_loss": 3.4639006622416915, + "tokens_seen": 1782251520 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046783838383838386, + "loss": 1.1682, + "theoretical_loss": 3.4638898001833183, + "tokens_seen": 1782317056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678181818181818, + "loss": 1.3635, + "theoretical_loss": 3.463878938636164, + "tokens_seen": 1782382592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046779797979797983, + "loss": 1.4278, + "theoretical_loss": 3.4638680776001856, + "tokens_seen": 1782448128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677777777777778, + "loss": 1.5485, + "theoretical_loss": 3.4638572170753408, + "tokens_seen": 1782513664 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046775757575757574, + "loss": 1.3842, + "theoretical_loss": 3.4638463570615854, + "tokens_seen": 1782579200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046773737373737376, + "loss": 1.3291, + "theoretical_loss": 3.463835497558878, + "tokens_seen": 1782644736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677171717171717, + "loss": 1.4354, + "theoretical_loss": 3.463824638567175, + "tokens_seen": 1782710272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676969696969697, + "loss": 1.3805, + "theoretical_loss": 3.4638137800864337, + "tokens_seen": 1782775808 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.41129031777381897, + "objective/train/avg_token_score": 0.034974388778209686, + "objective/train/docs_used": 1009251, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3251190185546875, + "objective/train/original_loss": 4.803771495819092, + "objective/train/theoretical_loss": 3.463802922116612, + "objective/train/tokens_used": 141700576, + "objective/train/unlikelihood_loss": 0.044761259108781815, + "theoretical_loss": 3.463802922116612, + "tokens_seen": 1782841344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676767676767677, + "loss": 1.3516, + "theoretical_loss": 3.463802922116612, + "tokens_seen": 1782841344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676565656565657, + "loss": 1.2101, + "theoretical_loss": 3.4637920646576656, + "tokens_seen": 1782906880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046763636363636365, + "loss": 1.4911, + "theoretical_loss": 3.463781207709553, + "tokens_seen": 1782972416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676161616161616, + "loss": 1.6088, + "theoretical_loss": 3.463770351272231, + "tokens_seen": 1783037952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675959595959596, + "loss": 1.377, + "theoretical_loss": 3.4637594953456565, + "tokens_seen": 1783103488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675757575757576, + "loss": 1.3993, + "theoretical_loss": 3.463748639929787, + "tokens_seen": 1783169024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675555555555556, + "loss": 1.4165, + "theoretical_loss": 3.4637377850245796, + "tokens_seen": 1783234560 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046753535353535354, + "loss": 1.3183, + "theoretical_loss": 3.4637269306299916, + "tokens_seen": 1783300096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675151515151515, + "loss": 1.436, + "theoretical_loss": 3.46371607674598, + "tokens_seen": 1783365632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674949494949495, + "loss": 1.3289, + "theoretical_loss": 3.463705223372502, + "tokens_seen": 1783431168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674747474747475, + "loss": 1.2814, + "theoretical_loss": 3.463694370509515, + "tokens_seen": 1783496704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674545454545455, + "loss": 1.4848, + "theoretical_loss": 3.4636835181569765, + "tokens_seen": 1783562240 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046743434343434343, + "loss": 1.3949, + "theoretical_loss": 3.4636726663148436, + "tokens_seen": 1783627776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674141414141414, + "loss": 1.2862, + "theoretical_loss": 3.463661814983073, + "tokens_seen": 1783693312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673939393939394, + "loss": 1.5041, + "theoretical_loss": 3.463650964161623, + "tokens_seen": 1783758848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673737373737374, + "loss": 1.3817, + "theoretical_loss": 3.4636401138504493, + "tokens_seen": 1783824384 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046735353535353537, + "loss": 1.6609, + "theoretical_loss": 3.4636292640495103, + "tokens_seen": 1783889920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673333333333333, + "loss": 1.2934, + "theoretical_loss": 3.4636184147587636, + "tokens_seen": 1783955456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673131313131313, + "loss": 1.3581, + "theoretical_loss": 3.4636075659781653, + "tokens_seen": 1784020992 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046729292929292935, + "loss": 1.3282, + "theoretical_loss": 3.4635967177076736, + "tokens_seen": 1784086528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672727272727273, + "loss": 1.4206, + "theoretical_loss": 3.463585869947245, + "tokens_seen": 1784152064 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046725252525252526, + "loss": 1.3386, + "theoretical_loss": 3.463575022696838, + "tokens_seen": 1784217600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672323232323232, + "loss": 1.4178, + "theoretical_loss": 3.4635641759564084, + "tokens_seen": 1784283136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672121212121212, + "loss": 1.427, + "theoretical_loss": 3.4635533297259142, + "tokens_seen": 1784348672 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046719191919191924, + "loss": 1.374, + "theoretical_loss": 3.463542484005313, + "tokens_seen": 1784414208 + }, + { + "epoch": 0.07, + "objective/train/aligned_token_count": 0.47012463212013245, + "objective/train/avg_token_score": 0.013481098227202892, + "objective/train/docs_used": 1010356, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.653877854347229, + "objective/train/original_loss": 4.1271233558654785, + "objective/train/theoretical_loss": 3.463531638794562, + "objective/train/tokens_used": 143338976, + "objective/train/unlikelihood_loss": 0.07333654910326004, + "theoretical_loss": 3.463531638794562, + "tokens_seen": 1784479744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004671717171717172, + "loss": 1.648, + "theoretical_loss": 3.463531638794562, + "tokens_seen": 1784479744 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046715151515151516, + "loss": 1.4924, + "theoretical_loss": 3.463520794093618, + "tokens_seen": 1784545280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671313131313131, + "loss": 1.4619, + "theoretical_loss": 3.4635099499024387, + "tokens_seen": 1784610816 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671111111111111, + "loss": 1.2438, + "theoretical_loss": 3.463499106220981, + "tokens_seen": 1784676352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046709090909090913, + "loss": 1.31, + "theoretical_loss": 3.463488263049203, + "tokens_seen": 1784741888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670707070707071, + "loss": 1.3666, + "theoretical_loss": 3.4634774203870617, + "tokens_seen": 1784807424 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046705050505050505, + "loss": 1.5564, + "theoretical_loss": 3.463466578234514, + "tokens_seen": 1784872960 + }, + { + "epoch": 0.08, + "learning_rate": 0.000467030303030303, + "loss": 1.513, + "theoretical_loss": 3.463455736591518, + "tokens_seen": 1784938496 + }, + { + "epoch": 0.08, + "learning_rate": 0.000467010101010101, + "loss": 1.4561, + "theoretical_loss": 3.4634448954580304, + "tokens_seen": 1785004032 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466989898989899, + "loss": 1.2265, + "theoretical_loss": 3.463434054834009, + "tokens_seen": 1785069568 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466969696969697, + "loss": 1.629, + "theoretical_loss": 3.463423214719411, + "tokens_seen": 1785135104 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046694949494949494, + "loss": 1.4004, + "theoretical_loss": 3.463412375114193, + "tokens_seen": 1785200640 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046692929292929295, + "loss": 1.5207, + "theoretical_loss": 3.4634015360183135, + "tokens_seen": 1785266176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669090909090909, + "loss": 1.6107, + "theoretical_loss": 3.4633906974317292, + "tokens_seen": 1785331712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668888888888889, + "loss": 1.0, + "theoretical_loss": 3.463379859354398, + "tokens_seen": 1785397248 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668686868686869, + "loss": 1.1273, + "theoretical_loss": 3.463369021786277, + "tokens_seen": 1785462784 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046684848484848483, + "loss": 1.2565, + "theoretical_loss": 3.463358184727323, + "tokens_seen": 1785528320 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046682828282828284, + "loss": 1.3508, + "theoretical_loss": 3.4633473481774946, + "tokens_seen": 1785593856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668080808080808, + "loss": 1.1729, + "theoretical_loss": 3.4633365121367485, + "tokens_seen": 1785659392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667878787878788, + "loss": 1.5044, + "theoretical_loss": 3.463325676605042, + "tokens_seen": 1785724928 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046676767676767677, + "loss": 1.5249, + "theoretical_loss": 3.4633148415823323, + "tokens_seen": 1785790464 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667474747474748, + "loss": 1.2512, + "theoretical_loss": 3.4633040070685777, + "tokens_seen": 1785856000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046672727272727274, + "loss": 1.6165, + "theoretical_loss": 3.463293173063735, + "tokens_seen": 1785921536 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667070707070707, + "loss": 1.3033, + "theoretical_loss": 3.463282339567762, + "tokens_seen": 1785987072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666868686868687, + "loss": 1.2939, + "theoretical_loss": 3.4632715065806154, + "tokens_seen": 1786052608 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.5376344323158264, + "objective/train/avg_token_score": 0.011802426539361477, + "objective/train/docs_used": 1010805, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8985878229141235, + "objective/train/original_loss": 3.866780996322632, + "objective/train/theoretical_loss": 3.463260674102253, + "objective/train/tokens_used": 144977376, + "objective/train/unlikelihood_loss": 0.08484289795160294, + "theoretical_loss": 3.463260674102253, + "tokens_seen": 1786118144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046666666666666666, + "loss": 1.5252, + "theoretical_loss": 3.463260674102253, + "tokens_seen": 1786118144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666464646464647, + "loss": 1.413, + "theoretical_loss": 3.4632498421326328, + "tokens_seen": 1786183680 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046662626262626263, + "loss": 1.3204, + "theoretical_loss": 3.4632390106717112, + "tokens_seen": 1786249216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666060606060606, + "loss": 1.5948, + "theoretical_loss": 3.4632281797194464, + "tokens_seen": 1786314752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665858585858586, + "loss": 1.5062, + "theoretical_loss": 3.463217349275796, + "tokens_seen": 1786380288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665656565656566, + "loss": 1.5023, + "theoretical_loss": 3.4632065193407167, + "tokens_seen": 1786445824 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046654545454545457, + "loss": 1.3644, + "theoretical_loss": 3.4631956899141665, + "tokens_seen": 1786511360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665252525252525, + "loss": 1.3282, + "theoretical_loss": 3.463184860996103, + "tokens_seen": 1786576896 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665050505050505, + "loss": 1.5257, + "theoretical_loss": 3.4631740325864833, + "tokens_seen": 1786642432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664848484848485, + "loss": 1.4772, + "theoretical_loss": 3.463163204685265, + "tokens_seen": 1786707968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664646464646465, + "loss": 1.2096, + "theoretical_loss": 3.4631523772924053, + "tokens_seen": 1786773504 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046644444444444446, + "loss": 1.583, + "theoretical_loss": 3.463141550407862, + "tokens_seen": 1786839040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664242424242424, + "loss": 1.3999, + "theoretical_loss": 3.4631307240315934, + "tokens_seen": 1786904576 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046640404040404037, + "loss": 1.535, + "theoretical_loss": 3.463119898163555, + "tokens_seen": 1786970112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663838383838384, + "loss": 1.4024, + "theoretical_loss": 3.463109072803706, + "tokens_seen": 1787035648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663636363636364, + "loss": 1.6384, + "theoretical_loss": 3.4630982479520034, + "tokens_seen": 1787101184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046634343434343435, + "loss": 1.4974, + "theoretical_loss": 3.4630874236084046, + "tokens_seen": 1787166720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663232323232323, + "loss": 1.3595, + "theoretical_loss": 3.4630765997728674, + "tokens_seen": 1787232256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663030303030303, + "loss": 1.2032, + "theoretical_loss": 3.463065776445349, + "tokens_seen": 1787297792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046628282828282833, + "loss": 1.6914, + "theoretical_loss": 3.463054953625807, + "tokens_seen": 1787363328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662626262626263, + "loss": 1.563, + "theoretical_loss": 3.4630441313141986, + "tokens_seen": 1787428864 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046624242424242424, + "loss": 1.6588, + "theoretical_loss": 3.4630333095104824, + "tokens_seen": 1787494400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662222222222222, + "loss": 1.3575, + "theoretical_loss": 3.463022488214615, + "tokens_seen": 1787559936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662020202020202, + "loss": 1.4825, + "theoretical_loss": 3.4630116674265548, + "tokens_seen": 1787625472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661818181818182, + "loss": 1.5337, + "theoretical_loss": 3.463000847146258, + "tokens_seen": 1787691008 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.27792033553123474, + "objective/train/avg_token_score": 0.018849870190024376, + "objective/train/docs_used": 1011887, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0499746799468994, + "objective/train/original_loss": 4.833069801330566, + "objective/train/theoretical_loss": 3.4629900273736833, + "objective/train/tokens_used": 146615776, + "objective/train/unlikelihood_loss": 0.06380809843540192, + "theoretical_loss": 3.4629900273736833, + "tokens_seen": 1787756544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661616161616162, + "loss": 1.4461, + "theoretical_loss": 3.4629900273736833, + "tokens_seen": 1787756544 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046614141414141414, + "loss": 1.2255, + "theoretical_loss": 3.462979208108788, + "tokens_seen": 1787822080 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661212121212121, + "loss": 1.6146, + "theoretical_loss": 3.462968389351529, + "tokens_seen": 1787887616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046610101010101016, + "loss": 1.5575, + "theoretical_loss": 3.4629575711018648, + "tokens_seen": 1787953152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004660808080808081, + "loss": 1.492, + "theoretical_loss": 3.462946753359753, + "tokens_seen": 1788018688 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046606060606060607, + "loss": 1.2284, + "theoretical_loss": 3.4629359361251506, + "tokens_seen": 1788084224 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046604040404040403, + "loss": 1.5239, + "theoretical_loss": 3.4629251193980153, + "tokens_seen": 1788149760 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466020202020202, + "loss": 1.4366, + "theoretical_loss": 3.462914303178305, + "tokens_seen": 1788215296 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046600000000000005, + "loss": 1.2893, + "theoretical_loss": 3.4629034874659768, + "tokens_seen": 1788280832 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465979797979798, + "loss": 1.5474, + "theoretical_loss": 3.4628926722609887, + "tokens_seen": 1788346368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046595959595959597, + "loss": 1.3654, + "theoretical_loss": 3.4628818575632985, + "tokens_seen": 1788411904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004659393939393939, + "loss": 1.4484, + "theoretical_loss": 3.4628710433728633, + "tokens_seen": 1788477440 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046591919191919193, + "loss": 1.4707, + "theoretical_loss": 3.462860229689641, + "tokens_seen": 1788542976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046589898989898994, + "loss": 1.5574, + "theoretical_loss": 3.4628494165135892, + "tokens_seen": 1788608512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658787878787879, + "loss": 1.3447, + "theoretical_loss": 3.4628386038446655, + "tokens_seen": 1788674048 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046585858585858586, + "loss": 1.4493, + "theoretical_loss": 3.4628277916828276, + "tokens_seen": 1788739584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658383838383838, + "loss": 1.5055, + "theoretical_loss": 3.462816980028033, + "tokens_seen": 1788805120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658181818181818, + "loss": 1.5176, + "theoretical_loss": 3.46280616888024, + "tokens_seen": 1788870656 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046579797979797984, + "loss": 1.3789, + "theoretical_loss": 3.462795358239405, + "tokens_seen": 1788936192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657777777777778, + "loss": 1.5674, + "theoretical_loss": 3.462784548105487, + "tokens_seen": 1789001728 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046575757575757575, + "loss": 1.3612, + "theoretical_loss": 3.4627737384784423, + "tokens_seen": 1789067264 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046573737373737376, + "loss": 1.4416, + "theoretical_loss": 3.46276292935823, + "tokens_seen": 1789132800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657171717171717, + "loss": 1.4044, + "theoretical_loss": 3.4627521207448066, + "tokens_seen": 1789198336 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046569696969696973, + "loss": 1.4033, + "theoretical_loss": 3.4627413126381303, + "tokens_seen": 1789263872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656767676767677, + "loss": 1.3574, + "theoretical_loss": 3.462730505038159, + "tokens_seen": 1789329408 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.47061339020729065, + "objective/train/avg_token_score": 0.010851036757230759, + "objective/train/docs_used": 1012559, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5306476354599, + "objective/train/original_loss": 4.064844608306885, + "objective/train/theoretical_loss": 3.46271969794485, + "objective/train/tokens_used": 148254176, + "objective/train/unlikelihood_loss": 0.07548725605010986, + "theoretical_loss": 3.46271969794485, + "tokens_seen": 1789394944 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046565656565656564, + "loss": 1.5801, + "theoretical_loss": 3.46271969794485, + "tokens_seen": 1789394944 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046563636363636365, + "loss": 1.2206, + "theoretical_loss": 3.4627088913581607, + "tokens_seen": 1789460480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656161616161616, + "loss": 1.3933, + "theoretical_loss": 3.4626980852780496, + "tokens_seen": 1789526016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655959595959596, + "loss": 1.4214, + "theoretical_loss": 3.4626872797044737, + "tokens_seen": 1789591552 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655757575757576, + "loss": 1.2427, + "theoretical_loss": 3.4626764746373913, + "tokens_seen": 1789657088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655555555555556, + "loss": 1.2924, + "theoretical_loss": 3.4626656700767597, + "tokens_seen": 1789722624 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046553535353535355, + "loss": 1.3831, + "theoretical_loss": 3.462654866022537, + "tokens_seen": 1789788160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655151515151515, + "loss": 1.4765, + "theoretical_loss": 3.4626440624746797, + "tokens_seen": 1789853696 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654949494949495, + "loss": 1.6304, + "theoretical_loss": 3.4626332594331473, + "tokens_seen": 1789919232 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046547474747474747, + "loss": 1.2434, + "theoretical_loss": 3.462622456897897, + "tokens_seen": 1789984768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654545454545455, + "loss": 1.2223, + "theoretical_loss": 3.4626116548688857, + "tokens_seen": 1790050304 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046543434343434344, + "loss": 1.5187, + "theoretical_loss": 3.462600853346072, + "tokens_seen": 1790115840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654141414141414, + "loss": 1.3788, + "theoretical_loss": 3.4625900523294133, + "tokens_seen": 1790181376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653939393939394, + "loss": 1.4454, + "theoretical_loss": 3.462579251818867, + "tokens_seen": 1790246912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653737373737374, + "loss": 1.3457, + "theoretical_loss": 3.4625684518143918, + "tokens_seen": 1790312448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653535353535354, + "loss": 1.4853, + "theoretical_loss": 3.4625576523159447, + "tokens_seen": 1790377984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046533333333333333, + "loss": 1.4036, + "theoretical_loss": 3.4625468533234836, + "tokens_seen": 1790443520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653131313131313, + "loss": 1.3071, + "theoretical_loss": 3.4625360548369666, + "tokens_seen": 1790509056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652929292929293, + "loss": 1.6316, + "theoretical_loss": 3.462525256856351, + "tokens_seen": 1790574592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652727272727273, + "loss": 1.2238, + "theoretical_loss": 3.462514459381595, + "tokens_seen": 1790640128 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046525252525252527, + "loss": 1.4982, + "theoretical_loss": 3.462503662412656, + "tokens_seen": 1790705664 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652323232323232, + "loss": 1.4581, + "theoretical_loss": 3.4624928659494927, + "tokens_seen": 1790771200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652121212121212, + "loss": 1.2027, + "theoretical_loss": 3.4624820699920615, + "tokens_seen": 1790836736 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046519191919191925, + "loss": 1.494, + "theoretical_loss": 3.462471274540321, + "tokens_seen": 1790902272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651717171717172, + "loss": 1.411, + "theoretical_loss": 3.4624604795942293, + "tokens_seen": 1790967808 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.4129398763179779, + "objective/train/avg_token_score": 0.012248078361153603, + "objective/train/docs_used": 1013169, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2744585275650024, + "objective/train/original_loss": 4.295300006866455, + "objective/train/theoretical_loss": 3.462449685153744, + "objective/train/tokens_used": 149892576, + "objective/train/unlikelihood_loss": 0.05737758427858353, + "theoretical_loss": 3.462449685153744, + "tokens_seen": 1791033344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046515151515151516, + "loss": 1.6501, + "theoretical_loss": 3.462449685153744, + "tokens_seen": 1791033344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651313131313131, + "loss": 1.6736, + "theoretical_loss": 3.462438891218822, + "tokens_seen": 1791098880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651111111111111, + "loss": 1.6972, + "theoretical_loss": 3.4624280977894224, + "tokens_seen": 1791164416 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046509090909090914, + "loss": 1.635, + "theoretical_loss": 3.462417304865502, + "tokens_seen": 1791229952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650707070707071, + "loss": 1.3991, + "theoretical_loss": 3.4624065124470196, + "tokens_seen": 1791295488 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046505050505050505, + "loss": 1.4326, + "theoretical_loss": 3.462395720533933, + "tokens_seen": 1791361024 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465030303030303, + "loss": 1.557, + "theoretical_loss": 3.462384929126199, + "tokens_seen": 1791426560 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046501010101010097, + "loss": 1.2343, + "theoretical_loss": 3.462374138223776, + "tokens_seen": 1791492096 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046498989898989903, + "loss": 1.5046, + "theoretical_loss": 3.4623633478266225, + "tokens_seen": 1791557632 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464969696969697, + "loss": 1.3209, + "theoretical_loss": 3.4623525579346954, + "tokens_seen": 1791623168 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046494949494949495, + "loss": 1.4695, + "theoretical_loss": 3.462341768547953, + "tokens_seen": 1791688704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649292929292929, + "loss": 1.2689, + "theoretical_loss": 3.462330979666353, + "tokens_seen": 1791754240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649090909090909, + "loss": 1.4558, + "theoretical_loss": 3.462320191289854, + "tokens_seen": 1791819776 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648888888888889, + "loss": 1.6189, + "theoretical_loss": 3.4623094034184128, + "tokens_seen": 1791885312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648686868686869, + "loss": 1.3666, + "theoretical_loss": 3.4622986160519877, + "tokens_seen": 1791950848 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046484848484848484, + "loss": 1.3568, + "theoretical_loss": 3.4622878291905366, + "tokens_seen": 1792016384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648282828282828, + "loss": 1.328, + "theoretical_loss": 3.4622770428340184, + "tokens_seen": 1792081920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648080808080808, + "loss": 1.4596, + "theoretical_loss": 3.4622662569823888, + "tokens_seen": 1792147456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647878787878788, + "loss": 1.5656, + "theoretical_loss": 3.4622554716356078, + "tokens_seen": 1792212992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647676767676768, + "loss": 1.5854, + "theoretical_loss": 3.462244686793632, + "tokens_seen": 1792278528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046474747474747473, + "loss": 1.4945, + "theoretical_loss": 3.46223390245642, + "tokens_seen": 1792344064 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046472727272727274, + "loss": 1.4186, + "theoretical_loss": 3.4622231186239296, + "tokens_seen": 1792409600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647070707070707, + "loss": 1.3961, + "theoretical_loss": 3.4622123352961185, + "tokens_seen": 1792475136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646868686868687, + "loss": 1.4866, + "theoretical_loss": 3.462201552472945, + "tokens_seen": 1792540672 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046466666666666667, + "loss": 1.4341, + "theoretical_loss": 3.4621907701543666, + "tokens_seen": 1792606208 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.4124511182308197, + "objective/train/avg_token_score": 0.007224504370242357, + "objective/train/docs_used": 1014342, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.416115403175354, + "objective/train/original_loss": 3.994509220123291, + "objective/train/theoretical_loss": 3.462179988340341, + "objective/train/tokens_used": 151530976, + "objective/train/unlikelihood_loss": 0.08687492460012436, + "theoretical_loss": 3.462179988340341, + "tokens_seen": 1792671744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646464646464646, + "loss": 1.4909, + "theoretical_loss": 3.462179988340341, + "tokens_seen": 1792671744 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046462626262626264, + "loss": 1.2383, + "theoretical_loss": 3.4621692070308274, + "tokens_seen": 1792737280 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046460606060606065, + "loss": 1.3036, + "theoretical_loss": 3.4621584262257823, + "tokens_seen": 1792802816 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645858585858586, + "loss": 1.3584, + "theoretical_loss": 3.4621476459251648, + "tokens_seen": 1792868352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046456565656565656, + "loss": 1.7118, + "theoretical_loss": 3.4621368661289322, + "tokens_seen": 1792933888 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046454545454545457, + "loss": 1.6058, + "theoretical_loss": 3.462126086837042, + "tokens_seen": 1792999424 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046452525252525253, + "loss": 1.2664, + "theoretical_loss": 3.4621153080494538, + "tokens_seen": 1793064960 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046450505050505054, + "loss": 1.3595, + "theoretical_loss": 3.462104529766124, + "tokens_seen": 1793130496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644848484848485, + "loss": 1.4667, + "theoretical_loss": 3.462093751987011, + "tokens_seen": 1793196032 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046446464646464645, + "loss": 1.4804, + "theoretical_loss": 3.4620829747120734, + "tokens_seen": 1793261568 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046444444444444446, + "loss": 1.7482, + "theoretical_loss": 3.4620721979412687, + "tokens_seen": 1793327104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644242424242424, + "loss": 1.3312, + "theoretical_loss": 3.462061421674554, + "tokens_seen": 1793392640 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046440404040404043, + "loss": 1.4292, + "theoretical_loss": 3.46205064591189, + "tokens_seen": 1793458176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643838383838384, + "loss": 1.4, + "theoretical_loss": 3.462039870653231, + "tokens_seen": 1793523712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643636363636364, + "loss": 1.3416, + "theoretical_loss": 3.4620290958985382, + "tokens_seen": 1793589248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046434343434343436, + "loss": 1.5423, + "theoretical_loss": 3.462018321647768, + "tokens_seen": 1793654784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643232323232323, + "loss": 1.3754, + "theoretical_loss": 3.462007547900879, + "tokens_seen": 1793720320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643030303030303, + "loss": 1.6072, + "theoretical_loss": 3.4619967746578286, + "tokens_seen": 1793785856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642828282828283, + "loss": 1.3222, + "theoretical_loss": 3.4619860019185755, + "tokens_seen": 1793851392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642626262626263, + "loss": 1.375, + "theoretical_loss": 3.4619752296830772, + "tokens_seen": 1793916928 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046424242424242425, + "loss": 1.3657, + "theoretical_loss": 3.461964457951292, + "tokens_seen": 1793982464 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642222222222222, + "loss": 1.4207, + "theoretical_loss": 3.461953686723178, + "tokens_seen": 1794048000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642020202020202, + "loss": 1.2671, + "theoretical_loss": 3.461942915998694, + "tokens_seen": 1794113536 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046418181818181823, + "loss": 1.4485, + "theoretical_loss": 3.4619321457777965, + "tokens_seen": 1794179072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641616161616162, + "loss": 1.4694, + "theoretical_loss": 3.4619213760604444, + "tokens_seen": 1794244608 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.4748900234699249, + "objective/train/avg_token_score": 0.019929148256778717, + "objective/train/docs_used": 1015045, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.42253839969635, + "objective/train/original_loss": 4.371203422546387, + "objective/train/theoretical_loss": 3.4619106068465957, + "objective/train/tokens_used": 153169376, + "objective/train/unlikelihood_loss": 0.07094332575798035, + "theoretical_loss": 3.4619106068465957, + "tokens_seen": 1794310144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046414141414141414, + "loss": 1.4498, + "theoretical_loss": 3.4619106068465957, + "tokens_seen": 1794310144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641212121212121, + "loss": 1.1417, + "theoretical_loss": 3.4618998381362087, + "tokens_seen": 1794375680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641010101010101, + "loss": 1.1921, + "theoretical_loss": 3.4618890699292413, + "tokens_seen": 1794441216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640808080808081, + "loss": 1.1941, + "theoretical_loss": 3.461878302225651, + "tokens_seen": 1794506752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640606060606061, + "loss": 1.4643, + "theoretical_loss": 3.4618675350253967, + "tokens_seen": 1794572288 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046404040404040403, + "loss": 1.4376, + "theoretical_loss": 3.4618567683284365, + "tokens_seen": 1794637824 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464020202020202, + "loss": 1.2493, + "theoretical_loss": 3.4618460021347275, + "tokens_seen": 1794703360 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046400000000000006, + "loss": 1.3667, + "theoretical_loss": 3.4618352364442293, + "tokens_seen": 1794768896 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463979797979798, + "loss": 1.4107, + "theoretical_loss": 3.4618244712568984, + "tokens_seen": 1794834432 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046395959595959597, + "loss": 1.3153, + "theoretical_loss": 3.4618137065726944, + "tokens_seen": 1794899968 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046393939393939393, + "loss": 1.3565, + "theoretical_loss": 3.4618029423915746, + "tokens_seen": 1794965504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639191919191919, + "loss": 1.4403, + "theoretical_loss": 3.461792178713497, + "tokens_seen": 1795031040 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046389898989898995, + "loss": 1.356, + "theoretical_loss": 3.46178141553842, + "tokens_seen": 1795096576 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638787878787879, + "loss": 1.1425, + "theoretical_loss": 3.461770652866302, + "tokens_seen": 1795162112 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046385858585858586, + "loss": 1.3359, + "theoretical_loss": 3.4617598906971008, + "tokens_seen": 1795227648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638383838383838, + "loss": 1.337, + "theoretical_loss": 3.4617491290307743, + "tokens_seen": 1795293184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046381818181818183, + "loss": 1.6086, + "theoretical_loss": 3.461738367867281, + "tokens_seen": 1795358720 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046379797979797984, + "loss": 1.4242, + "theoretical_loss": 3.461727607206579, + "tokens_seen": 1795424256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637777777777778, + "loss": 1.5684, + "theoretical_loss": 3.461716847048627, + "tokens_seen": 1795489792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046375757575757576, + "loss": 1.4422, + "theoretical_loss": 3.461706087393382, + "tokens_seen": 1795555328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637373737373737, + "loss": 1.6148, + "theoretical_loss": 3.461695328240803, + "tokens_seen": 1795620864 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637171717171717, + "loss": 1.0624, + "theoretical_loss": 3.461684569590848, + "tokens_seen": 1795686400 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046369696969696974, + "loss": 1.5246, + "theoretical_loss": 3.461673811443475, + "tokens_seen": 1795751936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636767676767677, + "loss": 1.4155, + "theoretical_loss": 3.4616630537986426, + "tokens_seen": 1795817472 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046365656565656565, + "loss": 1.3022, + "theoretical_loss": 3.4616522966563084, + "tokens_seen": 1795883008 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.38929617404937744, + "objective/train/avg_token_score": 0.010316792875528336, + "objective/train/docs_used": 1016473, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3159542083740234, + "objective/train/original_loss": 4.452950954437256, + "objective/train/theoretical_loss": 3.461641540016431, + "objective/train/tokens_used": 154807776, + "objective/train/unlikelihood_loss": 0.060373466461896896, + "theoretical_loss": 3.461641540016431, + "tokens_seen": 1795948544 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046363636363636366, + "loss": 1.411, + "theoretical_loss": 3.461641540016431, + "tokens_seen": 1795948544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636161616161616, + "loss": 1.3186, + "theoretical_loss": 3.4616307838789684, + "tokens_seen": 1796014080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046359595959595963, + "loss": 1.4337, + "theoretical_loss": 3.4616200282438783, + "tokens_seen": 1796079616 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635757575757576, + "loss": 1.3708, + "theoretical_loss": 3.4616092731111205, + "tokens_seen": 1796145152 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046355555555555554, + "loss": 1.41, + "theoretical_loss": 3.4615985184806517, + "tokens_seen": 1796210688 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046353535353535355, + "loss": 1.3782, + "theoretical_loss": 3.461587764352431, + "tokens_seen": 1796276224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635151515151515, + "loss": 1.3656, + "theoretical_loss": 3.4615770107264154, + "tokens_seen": 1796341760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634949494949495, + "loss": 1.314, + "theoretical_loss": 3.461566257602565, + "tokens_seen": 1796407296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634747474747475, + "loss": 1.2375, + "theoretical_loss": 3.4615555049808364, + "tokens_seen": 1796472832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634545454545455, + "loss": 1.5319, + "theoretical_loss": 3.4615447528611885, + "tokens_seen": 1796538368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046343434343434345, + "loss": 1.5685, + "theoretical_loss": 3.46153400124358, + "tokens_seen": 1796603904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634141414141414, + "loss": 1.321, + "theoretical_loss": 3.461523250127968, + "tokens_seen": 1796669440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633939393939394, + "loss": 1.2743, + "theoretical_loss": 3.4615124995143116, + "tokens_seen": 1796734976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046337373737373737, + "loss": 1.3477, + "theoretical_loss": 3.4615017494025686, + "tokens_seen": 1796800512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633535353535354, + "loss": 1.7626, + "theoretical_loss": 3.461490999792698, + "tokens_seen": 1796866048 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046333333333333334, + "loss": 1.7518, + "theoretical_loss": 3.461480250684657, + "tokens_seen": 1796931584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633131313131313, + "loss": 1.3579, + "theoretical_loss": 3.4614695020784048, + "tokens_seen": 1796997120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632929292929293, + "loss": 1.2737, + "theoretical_loss": 3.4614587539738992, + "tokens_seen": 1797062656 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632727272727273, + "loss": 1.5971, + "theoretical_loss": 3.4614480063710986, + "tokens_seen": 1797128192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632525252525253, + "loss": 1.3421, + "theoretical_loss": 3.461437259269961, + "tokens_seen": 1797193728 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046323232323232323, + "loss": 1.3835, + "theoretical_loss": 3.4614265126704455, + "tokens_seen": 1797259264 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632121212121212, + "loss": 1.3964, + "theoretical_loss": 3.46141576657251, + "tokens_seen": 1797324800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631919191919192, + "loss": 1.3049, + "theoretical_loss": 3.461405020976112, + "tokens_seen": 1797390336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631717171717172, + "loss": 1.5157, + "theoretical_loss": 3.4613942758812106, + "tokens_seen": 1797455872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046315151515151517, + "loss": 1.4886, + "theoretical_loss": 3.4613835312877645, + "tokens_seen": 1797521408 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.45002442598342896, + "objective/train/avg_token_score": 0.027033532038331032, + "objective/train/docs_used": 1017203, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.404165506362915, + "objective/train/original_loss": 4.197830677032471, + "objective/train/theoretical_loss": 3.461372787195731, + "objective/train/tokens_used": 156446176, + "objective/train/unlikelihood_loss": 0.05572577565908432, + "theoretical_loss": 3.461372787195731, + "tokens_seen": 1797586944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631313131313131, + "loss": 1.5094, + "theoretical_loss": 3.461372787195731, + "tokens_seen": 1797586944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631111111111111, + "loss": 1.0144, + "theoretical_loss": 3.461362043605069, + "tokens_seen": 1797652480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630909090909091, + "loss": 1.6355, + "theoretical_loss": 3.4613513005157373, + "tokens_seen": 1797718016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630707070707071, + "loss": 1.6244, + "theoretical_loss": 3.4613405579276932, + "tokens_seen": 1797783552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046305050505050506, + "loss": 1.4438, + "theoretical_loss": 3.4613298158408954, + "tokens_seen": 1797849088 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463030303030303, + "loss": 1.1163, + "theoretical_loss": 3.461319074255303, + "tokens_seen": 1797914624 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046301010101010097, + "loss": 1.3346, + "theoretical_loss": 3.461308333170873, + "tokens_seen": 1797980160 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046298989898989904, + "loss": 1.5162, + "theoretical_loss": 3.461297592587565, + "tokens_seen": 1798045696 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462969696969697, + "loss": 1.3135, + "theoretical_loss": 3.461286852505336, + "tokens_seen": 1798111232 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046294949494949495, + "loss": 1.5227, + "theoretical_loss": 3.4612761129241463, + "tokens_seen": 1798176768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629292929292929, + "loss": 1.4748, + "theoretical_loss": 3.4612653738439523, + "tokens_seen": 1798242304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629090909090909, + "loss": 1.2214, + "theoretical_loss": 3.4612546352647136, + "tokens_seen": 1798307840 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046288888888888893, + "loss": 1.1131, + "theoretical_loss": 3.461243897186388, + "tokens_seen": 1798373376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628686868686869, + "loss": 1.3627, + "theoretical_loss": 3.461233159608934, + "tokens_seen": 1798438912 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046284848484848484, + "loss": 1.5596, + "theoretical_loss": 3.4612224225323103, + "tokens_seen": 1798504448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628282828282828, + "loss": 1.3134, + "theoretical_loss": 3.461211685956475, + "tokens_seen": 1798569984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046280808080808087, + "loss": 1.3904, + "theoretical_loss": 3.4612009498813867, + "tokens_seen": 1798635520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627878787878788, + "loss": 1.3134, + "theoretical_loss": 3.4611902143070035, + "tokens_seen": 1798701056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627676767676768, + "loss": 1.2739, + "theoretical_loss": 3.4611794792332837, + "tokens_seen": 1798766592 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046274747474747474, + "loss": 1.2228, + "theoretical_loss": 3.4611687446601866, + "tokens_seen": 1798832128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627272727272727, + "loss": 1.2976, + "theoretical_loss": 3.4611580105876696, + "tokens_seen": 1798897664 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046270707070707076, + "loss": 1.3079, + "theoretical_loss": 3.4611472770156917, + "tokens_seen": 1798963200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626868686868687, + "loss": 1.3461, + "theoretical_loss": 3.4611365439442103, + "tokens_seen": 1799028736 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626666666666667, + "loss": 1.4401, + "theoretical_loss": 3.4611258113731855, + "tokens_seen": 1799094272 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046264646464646463, + "loss": 1.3903, + "theoretical_loss": 3.4611150793025747, + "tokens_seen": 1799159808 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.5161290168762207, + "objective/train/avg_token_score": 0.012231235392391682, + "objective/train/docs_used": 1018428, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5293020009994507, + "objective/train/original_loss": 3.6187143325805664, + "objective/train/theoretical_loss": 3.461104347732337, + "objective/train/tokens_used": 158084576, + "objective/train/unlikelihood_loss": 0.06525947153568268, + "theoretical_loss": 3.461104347732337, + "tokens_seen": 1799225344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046262626262626264, + "loss": 1.4842, + "theoretical_loss": 3.461104347732337, + "tokens_seen": 1799225344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046260606060606065, + "loss": 1.5779, + "theoretical_loss": 3.4610936166624295, + "tokens_seen": 1799290880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625858585858586, + "loss": 1.5093, + "theoretical_loss": 3.4610828860928122, + "tokens_seen": 1799356416 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046256565656565657, + "loss": 1.3926, + "theoretical_loss": 3.4610721560234428, + "tokens_seen": 1799421952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625454545454545, + "loss": 1.3863, + "theoretical_loss": 3.46106142645428, + "tokens_seen": 1799487488 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046252525252525253, + "loss": 1.3946, + "theoretical_loss": 3.4610506973852813, + "tokens_seen": 1799553024 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046250505050505054, + "loss": 1.5555, + "theoretical_loss": 3.4610399688164066, + "tokens_seen": 1799618560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624848484848485, + "loss": 1.5781, + "theoretical_loss": 3.4610292407476138, + "tokens_seen": 1799684096 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046246464646464646, + "loss": 1.3303, + "theoretical_loss": 3.4610185131788613, + "tokens_seen": 1799749632 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046244444444444447, + "loss": 1.4305, + "theoretical_loss": 3.461007786110108, + "tokens_seen": 1799815168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624242424242424, + "loss": 1.3423, + "theoretical_loss": 3.4609970595413113, + "tokens_seen": 1799880704 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046240404040404044, + "loss": 1.2689, + "theoretical_loss": 3.4609863334724307, + "tokens_seen": 1799946240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623838383838384, + "loss": 1.3048, + "theoretical_loss": 3.460975607903425, + "tokens_seen": 1800011776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046236363636363635, + "loss": 1.4274, + "theoretical_loss": 3.460964882834251, + "tokens_seen": 1800077312 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046234343434343436, + "loss": 1.4462, + "theoretical_loss": 3.460954158264869, + "tokens_seen": 1800142848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623232323232323, + "loss": 1.5049, + "theoretical_loss": 3.460943434195237, + "tokens_seen": 1800208384 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046230303030303033, + "loss": 1.5564, + "theoretical_loss": 3.460932710625313, + "tokens_seen": 1800273920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622828282828283, + "loss": 1.6183, + "theoretical_loss": 3.4609219875550563, + "tokens_seen": 1800339456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622626262626263, + "loss": 1.3066, + "theoretical_loss": 3.460911264984425, + "tokens_seen": 1800404992 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046224242424242425, + "loss": 1.4609, + "theoretical_loss": 3.460900542913377, + "tokens_seen": 1800470528 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622222222222222, + "loss": 1.0031, + "theoretical_loss": 3.460889821341872, + "tokens_seen": 1800536064 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622020202020202, + "loss": 1.5134, + "theoretical_loss": 3.4608791002698682, + "tokens_seen": 1800601600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621818181818182, + "loss": 1.3774, + "theoretical_loss": 3.460868379697324, + "tokens_seen": 1800667136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621616161616162, + "loss": 1.4704, + "theoretical_loss": 3.460857659624198, + "tokens_seen": 1800732672 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046214141414141415, + "loss": 1.6048, + "theoretical_loss": 3.4608469400504482, + "tokens_seen": 1800798208 + }, + { + "epoch": 0.08, + "objective/train/aligned_token_count": 0.4208821952342987, + "objective/train/avg_token_score": 0.024829581379890442, + "objective/train/docs_used": 1018922, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4344236850738525, + "objective/train/original_loss": 4.255611419677734, + "objective/train/theoretical_loss": 3.4608362209760335, + "objective/train/tokens_used": 159722976, + "objective/train/unlikelihood_loss": 0.07284124940633774, + "theoretical_loss": 3.4608362209760335, + "tokens_seen": 1800863744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004621212121212121, + "loss": 1.3354, + "theoretical_loss": 3.4608362209760335, + "tokens_seen": 1800863744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004621010101010101, + "loss": 1.73, + "theoretical_loss": 3.4608255024009136, + "tokens_seen": 1800929280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620808080808081, + "loss": 1.5529, + "theoretical_loss": 3.4608147843250454, + "tokens_seen": 1800994816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620606060606061, + "loss": 1.3241, + "theoretical_loss": 3.4608040667483886, + "tokens_seen": 1801060352 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046204040404040404, + "loss": 1.3847, + "theoretical_loss": 3.460793349670901, + "tokens_seen": 1801125888 + }, + { + "epoch": 0.09, + "learning_rate": 0.000462020202020202, + "loss": 1.603, + "theoretical_loss": 3.460782633092542, + "tokens_seen": 1801191424 + }, + { + "epoch": 0.09, + "learning_rate": 0.000462, + "loss": 1.4548, + "theoretical_loss": 3.4607719170132696, + "tokens_seen": 1801256960 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461979797979798, + "loss": 1.5245, + "theoretical_loss": 3.4607612014330424, + "tokens_seen": 1801322496 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461959595959596, + "loss": 1.2273, + "theoretical_loss": 3.4607504863518193, + "tokens_seen": 1801388032 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046193939393939393, + "loss": 1.6071, + "theoretical_loss": 3.460739771769559, + "tokens_seen": 1801453568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619191919191919, + "loss": 1.4026, + "theoretical_loss": 3.46072905768622, + "tokens_seen": 1801519104 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046189898989898996, + "loss": 1.4037, + "theoretical_loss": 3.4607183441017604, + "tokens_seen": 1801584640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618787878787879, + "loss": 1.3166, + "theoretical_loss": 3.460707631016139, + "tokens_seen": 1801650176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046185858585858587, + "loss": 1.3045, + "theoretical_loss": 3.4606969184293153, + "tokens_seen": 1801715712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618383838383838, + "loss": 1.7083, + "theoretical_loss": 3.460686206341247, + "tokens_seen": 1801781248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618181818181818, + "loss": 1.531, + "theoretical_loss": 3.4606754947518934, + "tokens_seen": 1801846784 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046179797979797985, + "loss": 1.3199, + "theoretical_loss": 3.4606647836612128, + "tokens_seen": 1801912320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617777777777778, + "loss": 1.3986, + "theoretical_loss": 3.460654073069163, + "tokens_seen": 1801977856 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046175757575757576, + "loss": 1.3911, + "theoretical_loss": 3.4606433629757043, + "tokens_seen": 1802043392 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617373737373737, + "loss": 1.279, + "theoretical_loss": 3.4606326533807943, + "tokens_seen": 1802108928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617171717171717, + "loss": 1.4534, + "theoretical_loss": 3.4606219442843917, + "tokens_seen": 1802174464 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046169696969696974, + "loss": 1.5236, + "theoretical_loss": 3.460611235686456, + "tokens_seen": 1802240000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616767676767677, + "loss": 1.5087, + "theoretical_loss": 3.4606005275869447, + "tokens_seen": 1802305536 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046165656565656565, + "loss": 1.4131, + "theoretical_loss": 3.460589819985817, + "tokens_seen": 1802371072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616363636363636, + "loss": 1.4507, + "theoretical_loss": 3.460579112883032, + "tokens_seen": 1802436608 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.33669355511665344, + "objective/train/avg_token_score": 0.009447429329156876, + "objective/train/docs_used": 1019593, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2540216445922852, + "objective/train/original_loss": 4.323973655700684, + "objective/train/theoretical_loss": 3.460568406278548, + "objective/train/tokens_used": 161361376, + "objective/train/unlikelihood_loss": 0.07911861687898636, + "theoretical_loss": 3.460568406278548, + "tokens_seen": 1802502144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616161616161616, + "loss": 1.3935, + "theoretical_loss": 3.460568406278548, + "tokens_seen": 1802502144 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046159595959595963, + "loss": 1.336, + "theoretical_loss": 3.4605577001723233, + "tokens_seen": 1802567680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615757575757576, + "loss": 1.4096, + "theoretical_loss": 3.460546994564317, + "tokens_seen": 1802633216 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046155555555555555, + "loss": 1.3497, + "theoretical_loss": 3.460536289454488, + "tokens_seen": 1802698752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615353535353535, + "loss": 1.5146, + "theoretical_loss": 3.460525584842795, + "tokens_seen": 1802764288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615151515151515, + "loss": 1.3389, + "theoretical_loss": 3.4605148807291966, + "tokens_seen": 1802829824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614949494949495, + "loss": 1.3034, + "theoretical_loss": 3.460504177113651, + "tokens_seen": 1802895360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614747474747475, + "loss": 1.542, + "theoretical_loss": 3.4604934739961175, + "tokens_seen": 1802960896 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046145454545454544, + "loss": 1.5322, + "theoretical_loss": 3.460482771376555, + "tokens_seen": 1803026432 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046143434343434345, + "loss": 1.4973, + "theoretical_loss": 3.460472069254922, + "tokens_seen": 1803091968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614141414141414, + "loss": 1.2724, + "theoretical_loss": 3.4604613676311766, + "tokens_seen": 1803157504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613939393939394, + "loss": 1.3662, + "theoretical_loss": 3.4604506665052783, + "tokens_seen": 1803223040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613737373737374, + "loss": 1.4956, + "theoretical_loss": 3.4604399658771863, + "tokens_seen": 1803288576 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046135353535353533, + "loss": 1.1918, + "theoretical_loss": 3.460429265746858, + "tokens_seen": 1803354112 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046133333333333334, + "loss": 1.2887, + "theoretical_loss": 3.460418566114253, + "tokens_seen": 1803419648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613131313131313, + "loss": 1.2515, + "theoretical_loss": 3.4604078669793297, + "tokens_seen": 1803485184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612929292929293, + "loss": 1.1434, + "theoretical_loss": 3.4603971683420474, + "tokens_seen": 1803550720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046127272727272727, + "loss": 1.5023, + "theoretical_loss": 3.4603864702023643, + "tokens_seen": 1803616256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612525252525253, + "loss": 1.495, + "theoretical_loss": 3.4603757725602398, + "tokens_seen": 1803681792 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046123232323232324, + "loss": 1.493, + "theoretical_loss": 3.4603650754156323, + "tokens_seen": 1803747328 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612121212121212, + "loss": 1.5071, + "theoretical_loss": 3.4603543787685, + "tokens_seen": 1803812864 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611919191919192, + "loss": 1.3662, + "theoretical_loss": 3.460343682618803, + "tokens_seen": 1803878400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046117171717171716, + "loss": 1.3852, + "theoretical_loss": 3.460332986966499, + "tokens_seen": 1803943936 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046115151515151517, + "loss": 1.3354, + "theoretical_loss": 3.4603222918115475, + "tokens_seen": 1804009472 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046113131313131313, + "loss": 1.2229, + "theoretical_loss": 3.4603115971539067, + "tokens_seen": 1804075008 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.36766862869262695, + "objective/train/avg_token_score": 0.01413616631180048, + "objective/train/docs_used": 1020148, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.183333396911621, + "objective/train/original_loss": 4.116270065307617, + "objective/train/theoretical_loss": 3.460300902993536, + "objective/train/tokens_used": 162999776, + "objective/train/unlikelihood_loss": 0.08400173485279083, + "theoretical_loss": 3.460300902993536, + "tokens_seen": 1804140544 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046111111111111114, + "loss": 1.4689, + "theoretical_loss": 3.460300902993536, + "tokens_seen": 1804140544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610909090909091, + "loss": 1.4244, + "theoretical_loss": 3.4602902093303936, + "tokens_seen": 1804206080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610707070707071, + "loss": 1.6157, + "theoretical_loss": 3.460279516164439, + "tokens_seen": 1804271616 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046105050505050506, + "loss": 1.3807, + "theoretical_loss": 3.460268823495631, + "tokens_seen": 1804337152 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461030303030303, + "loss": 1.3098, + "theoretical_loss": 3.4602581313239273, + "tokens_seen": 1804402688 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046101010101010103, + "loss": 1.4589, + "theoretical_loss": 3.460247439649288, + "tokens_seen": 1804468224 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460989898989899, + "loss": 1.2463, + "theoretical_loss": 3.4602367484716714, + "tokens_seen": 1804533760 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460969696969697, + "loss": 1.5439, + "theoretical_loss": 3.4602260577910364, + "tokens_seen": 1804599296 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046094949494949496, + "loss": 1.3021, + "theoretical_loss": 3.4602153676073417, + "tokens_seen": 1804664832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609292929292929, + "loss": 1.4273, + "theoretical_loss": 3.4602046779205464, + "tokens_seen": 1804730368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609090909090909, + "loss": 1.587, + "theoretical_loss": 3.4601939887306097, + "tokens_seen": 1804795904 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046088888888888894, + "loss": 1.2694, + "theoretical_loss": 3.4601833000374898, + "tokens_seen": 1804861440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608686868686869, + "loss": 1.3032, + "theoretical_loss": 3.4601726118411458, + "tokens_seen": 1804926976 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046084848484848485, + "loss": 1.4727, + "theoretical_loss": 3.460161924141537, + "tokens_seen": 1804992512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608282828282828, + "loss": 1.4011, + "theoretical_loss": 3.4601512369386214, + "tokens_seen": 1805058048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608080808080808, + "loss": 1.2257, + "theoretical_loss": 3.4601405502323583, + "tokens_seen": 1805123584 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046078787878787883, + "loss": 1.373, + "theoretical_loss": 3.460129864022707, + "tokens_seen": 1805189120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607676767676768, + "loss": 1.2235, + "theoretical_loss": 3.460119178309626, + "tokens_seen": 1805254656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046074747474747474, + "loss": 1.5574, + "theoretical_loss": 3.4601084930930743, + "tokens_seen": 1805320192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607272727272727, + "loss": 1.3258, + "theoretical_loss": 3.4600978083730105, + "tokens_seen": 1805385728 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046070707070707077, + "loss": 1.4703, + "theoretical_loss": 3.460087124149394, + "tokens_seen": 1805451264 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606868686868687, + "loss": 1.2853, + "theoretical_loss": 3.4600764404221835, + "tokens_seen": 1805516800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606666666666667, + "loss": 1.5211, + "theoretical_loss": 3.460065757191338, + "tokens_seen": 1805582336 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046064646464646464, + "loss": 1.5547, + "theoretical_loss": 3.460055074456816, + "tokens_seen": 1805647872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606262626262626, + "loss": 1.3416, + "theoretical_loss": 3.4600443922185766, + "tokens_seen": 1805713408 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.40169844031333923, + "objective/train/avg_token_score": 0.0685867890715599, + "objective/train/docs_used": 1021470, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3303582668304443, + "objective/train/original_loss": 4.088999271392822, + "objective/train/theoretical_loss": 3.4600337104765795, + "objective/train/tokens_used": 164638176, + "objective/train/unlikelihood_loss": 0.06166958063840866, + "theoretical_loss": 3.4600337104765795, + "tokens_seen": 1805778944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046060606060606066, + "loss": 1.422, + "theoretical_loss": 3.4600337104765795, + "tokens_seen": 1805778944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605858585858586, + "loss": 1.3202, + "theoretical_loss": 3.4600230292307828, + "tokens_seen": 1805844480 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046056565656565657, + "loss": 1.2661, + "theoretical_loss": 3.4600123484811456, + "tokens_seen": 1805910016 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046054545454545453, + "loss": 1.5477, + "theoretical_loss": 3.460001668227627, + "tokens_seen": 1805975552 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046052525252525254, + "loss": 1.2371, + "theoretical_loss": 3.459990988470186, + "tokens_seen": 1806041088 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046050505050505055, + "loss": 1.3883, + "theoretical_loss": 3.459980309208781, + "tokens_seen": 1806106624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604848484848485, + "loss": 1.4474, + "theoretical_loss": 3.459969630443372, + "tokens_seen": 1806172160 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046046464646464646, + "loss": 1.4572, + "theoretical_loss": 3.459958952173917, + "tokens_seen": 1806237696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604444444444444, + "loss": 1.3687, + "theoretical_loss": 3.459948274400375, + "tokens_seen": 1806303232 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046042424242424243, + "loss": 1.5883, + "theoretical_loss": 3.459937597122706, + "tokens_seen": 1806368768 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046040404040404044, + "loss": 1.113, + "theoretical_loss": 3.459926920340868, + "tokens_seen": 1806434304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603838383838384, + "loss": 1.3119, + "theoretical_loss": 3.4599162440548197, + "tokens_seen": 1806499840 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046036363636363636, + "loss": 1.1213, + "theoretical_loss": 3.4599055682645217, + "tokens_seen": 1806565376 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046034343434343437, + "loss": 1.5036, + "theoretical_loss": 3.459894892969931, + "tokens_seen": 1806630912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603232323232323, + "loss": 1.4614, + "theoretical_loss": 3.459884218171008, + "tokens_seen": 1806696448 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046030303030303034, + "loss": 1.2264, + "theoretical_loss": 3.4598735438677117, + "tokens_seen": 1806761984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602828282828283, + "loss": 1.2862, + "theoretical_loss": 3.4598628700600003, + "tokens_seen": 1806827520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046026262626262625, + "loss": 1.4969, + "theoretical_loss": 3.4598521967478337, + "tokens_seen": 1806893056 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046024242424242426, + "loss": 1.3143, + "theoretical_loss": 3.4598415239311695, + "tokens_seen": 1806958592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602222222222222, + "loss": 1.3717, + "theoretical_loss": 3.4598308516099685, + "tokens_seen": 1807024128 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046020202020202023, + "loss": 1.2754, + "theoretical_loss": 3.4598201797841885, + "tokens_seen": 1807089664 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601818181818182, + "loss": 1.3765, + "theoretical_loss": 3.459809508453789, + "tokens_seen": 1807155200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601616161616162, + "loss": 1.4214, + "theoretical_loss": 3.459798837618729, + "tokens_seen": 1807220736 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046014141414141415, + "loss": 1.2824, + "theoretical_loss": 3.4597881672789677, + "tokens_seen": 1807286272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601212121212121, + "loss": 1.2776, + "theoretical_loss": 3.4597774974344637, + "tokens_seen": 1807351808 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.5522360801696777, + "objective/train/avg_token_score": 0.005671007093042135, + "objective/train/docs_used": 1022105, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.969338059425354, + "objective/train/original_loss": 3.9141592979431152, + "objective/train/theoretical_loss": 3.4597668280851765, + "objective/train/tokens_used": 166276576, + "objective/train/unlikelihood_loss": 0.07195361703634262, + "theoretical_loss": 3.4597668280851765, + "tokens_seen": 1807417344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601010101010101, + "loss": 1.4951, + "theoretical_loss": 3.4597668280851765, + "tokens_seen": 1807417344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600808080808081, + "loss": 1.4208, + "theoretical_loss": 3.4597561592310644, + "tokens_seen": 1807482880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600606060606061, + "loss": 1.4108, + "theoretical_loss": 3.4597454908720877, + "tokens_seen": 1807548416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046004040404040405, + "loss": 1.5542, + "theoretical_loss": 3.459734823008205, + "tokens_seen": 1807613952 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460020202020202, + "loss": 1.2047, + "theoretical_loss": 3.4597241556393747, + "tokens_seen": 1807679488 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046, + "loss": 1.4753, + "theoretical_loss": 3.4597134887655567, + "tokens_seen": 1807745024 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459979797979798, + "loss": 1.4563, + "theoretical_loss": 3.459702822386709, + "tokens_seen": 1807810560 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459959595959596, + "loss": 1.4975, + "theoretical_loss": 3.4596921565027925, + "tokens_seen": 1807876096 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045993939393939394, + "loss": 1.5808, + "theoretical_loss": 3.4596814911137646, + "tokens_seen": 1807941632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599191919191919, + "loss": 1.3581, + "theoretical_loss": 3.459670826219585, + "tokens_seen": 1808007168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598989898989899, + "loss": 1.4058, + "theoretical_loss": 3.459660161820213, + "tokens_seen": 1808072704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598787878787879, + "loss": 1.4865, + "theoretical_loss": 3.459649497915608, + "tokens_seen": 1808138240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598585858585859, + "loss": 1.3384, + "theoretical_loss": 3.459638834505728, + "tokens_seen": 1808203776 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045983838383838383, + "loss": 1.2745, + "theoretical_loss": 3.4596281715905333, + "tokens_seen": 1808269312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598181818181818, + "loss": 1.2995, + "theoretical_loss": 3.4596175091699823, + "tokens_seen": 1808334848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597979797979798, + "loss": 1.2344, + "theoretical_loss": 3.459606847244034, + "tokens_seen": 1808400384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597777777777778, + "loss": 1.2524, + "theoretical_loss": 3.4595961858126483, + "tokens_seen": 1808465920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045975757575757577, + "loss": 1.3612, + "theoretical_loss": 3.459585524875784, + "tokens_seen": 1808531456 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597373737373737, + "loss": 1.3054, + "theoretical_loss": 3.4595748644334, + "tokens_seen": 1808596992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597171717171717, + "loss": 1.4338, + "theoretical_loss": 3.4595642044854555, + "tokens_seen": 1808662528 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045969696969696975, + "loss": 1.3573, + "theoretical_loss": 3.45955354503191, + "tokens_seen": 1808728064 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596767676767677, + "loss": 1.2754, + "theoretical_loss": 3.4595428860727218, + "tokens_seen": 1808793600 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045965656565656566, + "loss": 1.45, + "theoretical_loss": 3.4595322276078515, + "tokens_seen": 1808859136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596363636363636, + "loss": 1.3942, + "theoretical_loss": 3.459521569637257, + "tokens_seen": 1808924672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596161616161616, + "loss": 1.5969, + "theoretical_loss": 3.459510912160898, + "tokens_seen": 1808990208 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.4491080045700073, + "objective/train/avg_token_score": 0.009337084367871284, + "objective/train/docs_used": 1022607, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.514532446861267, + "objective/train/original_loss": 3.918682336807251, + "objective/train/theoretical_loss": 3.4595002551787335, + "objective/train/tokens_used": 167914976, + "objective/train/unlikelihood_loss": 0.08269507437944412, + "theoretical_loss": 3.4595002551787335, + "tokens_seen": 1809055744 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045959595959595964, + "loss": 1.38, + "theoretical_loss": 3.4595002551787335, + "tokens_seen": 1809055744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595757575757576, + "loss": 1.4447, + "theoretical_loss": 3.4594895986907233, + "tokens_seen": 1809121280 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045955555555555555, + "loss": 1.2231, + "theoretical_loss": 3.4594789426968253, + "tokens_seen": 1809186816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595353535353535, + "loss": 1.5118, + "theoretical_loss": 3.4594682871969997, + "tokens_seen": 1809252352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595151515151515, + "loss": 1.3982, + "theoretical_loss": 3.4594576321912056, + "tokens_seen": 1809317888 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045949494949494953, + "loss": 1.3473, + "theoretical_loss": 3.459446977679402, + "tokens_seen": 1809383424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594747474747475, + "loss": 1.4469, + "theoretical_loss": 3.4594363236615484, + "tokens_seen": 1809448960 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045945454545454545, + "loss": 1.1953, + "theoretical_loss": 3.4594256701376036, + "tokens_seen": 1809514496 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594343434343434, + "loss": 1.4947, + "theoretical_loss": 3.459415017107527, + "tokens_seen": 1809580032 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045941414141414147, + "loss": 1.1828, + "theoretical_loss": 3.4594043645712773, + "tokens_seen": 1809645568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593939393939394, + "loss": 1.5579, + "theoretical_loss": 3.459393712528815, + "tokens_seen": 1809711104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593737373737374, + "loss": 1.2093, + "theoretical_loss": 3.459383060980098, + "tokens_seen": 1809776640 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045935353535353534, + "loss": 1.3925, + "theoretical_loss": 3.459372409925087, + "tokens_seen": 1809842176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045933333333333335, + "loss": 1.5784, + "theoretical_loss": 3.4593617593637394, + "tokens_seen": 1809907712 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045931313131313136, + "loss": 1.4518, + "theoretical_loss": 3.4593511092960156, + "tokens_seen": 1809973248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592929292929293, + "loss": 1.2988, + "theoretical_loss": 3.459340459721875, + "tokens_seen": 1810038784 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592727272727273, + "loss": 1.4289, + "theoretical_loss": 3.459329810641276, + "tokens_seen": 1810104320 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045925252525252523, + "loss": 1.4281, + "theoretical_loss": 3.4593191620541783, + "tokens_seen": 1810169856 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045923232323232324, + "loss": 1.2985, + "theoretical_loss": 3.4593085139605417, + "tokens_seen": 1810235392 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045921212121212125, + "loss": 1.3204, + "theoretical_loss": 3.459297866360325, + "tokens_seen": 1810300928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004591919191919192, + "loss": 1.4494, + "theoretical_loss": 3.459287219253487, + "tokens_seen": 1810366464 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045917171717171717, + "loss": 1.5598, + "theoretical_loss": 3.459276572639988, + "tokens_seen": 1810432000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004591515151515152, + "loss": 1.3901, + "theoretical_loss": 3.459265926519786, + "tokens_seen": 1810497536 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045913131313131313, + "loss": 1.3048, + "theoretical_loss": 3.459255280892841, + "tokens_seen": 1810563072 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045911111111111115, + "loss": 1.4061, + "theoretical_loss": 3.459244635759113, + "tokens_seen": 1810628608 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.2624022364616394, + "objective/train/avg_token_score": 0.027153268456459045, + "objective/train/docs_used": 1024104, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0158097743988037, + "objective/train/original_loss": 5.203606128692627, + "objective/train/theoretical_loss": 3.459233991118561, + "objective/train/tokens_used": 169553376, + "objective/train/unlikelihood_loss": 0.043073952198028564, + "theoretical_loss": 3.459233991118561, + "tokens_seen": 1810694144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004590909090909091, + "loss": 1.26, + "theoretical_loss": 3.459233991118561, + "tokens_seen": 1810694144 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045907070707070706, + "loss": 1.2706, + "theoretical_loss": 3.4592233469711426, + "tokens_seen": 1810759680 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045905050505050507, + "loss": 1.0934, + "theoretical_loss": 3.459212703316819, + "tokens_seen": 1810825216 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045903030303030303, + "loss": 1.3351, + "theoretical_loss": 3.4592020601555493, + "tokens_seen": 1810890752 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045901010101010104, + "loss": 1.2675, + "theoretical_loss": 3.4591914174872915, + "tokens_seen": 1810956288 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458989898989899, + "loss": 1.3725, + "theoretical_loss": 3.4591807753120065, + "tokens_seen": 1811021824 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458969696969697, + "loss": 1.4893, + "theoretical_loss": 3.4591701336296534, + "tokens_seen": 1811087360 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045894949494949496, + "loss": 1.347, + "theoretical_loss": 3.4591594924401905, + "tokens_seen": 1811152896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589292929292929, + "loss": 1.4972, + "theoretical_loss": 3.459148851743578, + "tokens_seen": 1811218432 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045890909090909093, + "loss": 1.2867, + "theoretical_loss": 3.459138211539775, + "tokens_seen": 1811283968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588888888888889, + "loss": 1.4704, + "theoretical_loss": 3.4591275718287404, + "tokens_seen": 1811349504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588686868686869, + "loss": 1.27, + "theoretical_loss": 3.459116932610435, + "tokens_seen": 1811415040 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045884848484848486, + "loss": 1.3998, + "theoretical_loss": 3.4591062938848163, + "tokens_seen": 1811480576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588282828282828, + "loss": 1.4274, + "theoretical_loss": 3.4590956556518444, + "tokens_seen": 1811546112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588080808080808, + "loss": 1.5243, + "theoretical_loss": 3.4590850179114794, + "tokens_seen": 1811611648 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045878787878787883, + "loss": 1.3508, + "theoretical_loss": 3.45907438066368, + "tokens_seen": 1811677184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587676767676768, + "loss": 1.4852, + "theoretical_loss": 3.4590637439084055, + "tokens_seen": 1811742720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045874747474747475, + "loss": 1.3903, + "theoretical_loss": 3.459053107645615, + "tokens_seen": 1811808256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587272727272727, + "loss": 1.5327, + "theoretical_loss": 3.459042471875269, + "tokens_seen": 1811873792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587070707070707, + "loss": 1.202, + "theoretical_loss": 3.4590318365973256, + "tokens_seen": 1811939328 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045868686868686873, + "loss": 1.2374, + "theoretical_loss": 3.459021201811745, + "tokens_seen": 1812004864 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586666666666667, + "loss": 1.3289, + "theoretical_loss": 3.4590105675184866, + "tokens_seen": 1812070400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045864646464646464, + "loss": 1.4467, + "theoretical_loss": 3.458999933717509, + "tokens_seen": 1812135936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586262626262626, + "loss": 1.1531, + "theoretical_loss": 3.4589893004087733, + "tokens_seen": 1812201472 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045860606060606066, + "loss": 1.4899, + "theoretical_loss": 3.4589786675922367, + "tokens_seen": 1812267008 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.4100073277950287, + "objective/train/avg_token_score": 0.012066290713846684, + "objective/train/docs_used": 1024786, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3427298069000244, + "objective/train/original_loss": 4.287503242492676, + "objective/train/theoretical_loss": 3.4589680352678602, + "objective/train/tokens_used": 171191776, + "objective/train/unlikelihood_loss": 0.05709630250930786, + "theoretical_loss": 3.4589680352678602, + "tokens_seen": 1812332544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585858585858586, + "loss": 1.2515, + "theoretical_loss": 3.4589680352678602, + "tokens_seen": 1812332544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585656565656566, + "loss": 1.2303, + "theoretical_loss": 3.4589574034356025, + "tokens_seen": 1812398080 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045854545454545453, + "loss": 1.6304, + "theoretical_loss": 3.4589467720954232, + "tokens_seen": 1812463616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585252525252525, + "loss": 1.3892, + "theoretical_loss": 3.4589361412472828, + "tokens_seen": 1812529152 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045850505050505056, + "loss": 1.4854, + "theoretical_loss": 3.4589255108911385, + "tokens_seen": 1812594688 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584848484848485, + "loss": 1.5415, + "theoretical_loss": 3.458914881026952, + "tokens_seen": 1812660224 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045846464646464647, + "loss": 1.4956, + "theoretical_loss": 3.458904251654681, + "tokens_seen": 1812725760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584444444444444, + "loss": 1.3248, + "theoretical_loss": 3.458893622774286, + "tokens_seen": 1812791296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584242424242424, + "loss": 1.3537, + "theoretical_loss": 3.4588829943857258, + "tokens_seen": 1812856832 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045840404040404045, + "loss": 1.4755, + "theoretical_loss": 3.4588723664889605, + "tokens_seen": 1812922368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583838383838384, + "loss": 1.2218, + "theoretical_loss": 3.4588617390839493, + "tokens_seen": 1812987904 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045836363636363636, + "loss": 1.3251, + "theoretical_loss": 3.4588511121706516, + "tokens_seen": 1813053440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583434343434343, + "loss": 1.4598, + "theoretical_loss": 3.458840485749027, + "tokens_seen": 1813118976 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045832323232323233, + "loss": 1.6122, + "theoretical_loss": 3.4588298598190352, + "tokens_seen": 1813184512 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045830303030303034, + "loss": 1.1923, + "theoretical_loss": 3.458819234380635, + "tokens_seen": 1813250048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582828282828283, + "loss": 1.3857, + "theoretical_loss": 3.4588086094337864, + "tokens_seen": 1813315584 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045826262626262626, + "loss": 1.4029, + "theoretical_loss": 3.4587979849784487, + "tokens_seen": 1813381120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582424242424242, + "loss": 1.3769, + "theoretical_loss": 3.458787361014582, + "tokens_seen": 1813446656 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582222222222222, + "loss": 1.3538, + "theoretical_loss": 3.4587767375421445, + "tokens_seen": 1813512192 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045820202020202023, + "loss": 1.2557, + "theoretical_loss": 3.458766114561097, + "tokens_seen": 1813577728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581818181818182, + "loss": 1.4583, + "theoretical_loss": 3.458755492071398, + "tokens_seen": 1813643264 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045816161616161615, + "loss": 1.3099, + "theoretical_loss": 3.458744870073008, + "tokens_seen": 1813708800 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045814141414141416, + "loss": 1.2544, + "theoretical_loss": 3.458734248565886, + "tokens_seen": 1813774336 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581212121212121, + "loss": 1.461, + "theoretical_loss": 3.4587236275499915, + "tokens_seen": 1813839872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581010101010101, + "loss": 1.1234, + "theoretical_loss": 3.458713007025284, + "tokens_seen": 1813905408 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.4995112419128418, + "objective/train/avg_token_score": 0.0071546295657753944, + "objective/train/docs_used": 1026014, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7556525468826294, + "objective/train/original_loss": 3.839704990386963, + "objective/train/theoretical_loss": 3.4587023869917237, + "objective/train/tokens_used": 172830176, + "objective/train/unlikelihood_loss": 0.09379331767559052, + "theoretical_loss": 3.4587023869917237, + "tokens_seen": 1813970944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580808080808081, + "loss": 1.4608, + "theoretical_loss": 3.4587023869917237, + "tokens_seen": 1813970944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045806060606060604, + "loss": 1.3605, + "theoretical_loss": 3.458691767449269, + "tokens_seen": 1814036480 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045804040404040405, + "loss": 1.1711, + "theoretical_loss": 3.45868114839788, + "tokens_seen": 1814102016 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458020202020202, + "loss": 1.3351, + "theoretical_loss": 3.4586705298375167, + "tokens_seen": 1814167552 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458, + "loss": 1.5054, + "theoretical_loss": 3.4586599117681383, + "tokens_seen": 1814233088 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457979797979798, + "loss": 1.6297, + "theoretical_loss": 3.458649294189704, + "tokens_seen": 1814298624 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457959595959596, + "loss": 1.4436, + "theoretical_loss": 3.458638677102174, + "tokens_seen": 1814364160 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045793939393939394, + "loss": 1.3552, + "theoretical_loss": 3.4586280605055073, + "tokens_seen": 1814429696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579191919191919, + "loss": 1.5167, + "theoretical_loss": 3.458617444399664, + "tokens_seen": 1814495232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578989898989899, + "loss": 1.4367, + "theoretical_loss": 3.4586068287846032, + "tokens_seen": 1814560768 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045787878787878787, + "loss": 1.4934, + "theoretical_loss": 3.458596213660285, + "tokens_seen": 1814626304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578585858585859, + "loss": 1.3668, + "theoretical_loss": 3.4585855990266685, + "tokens_seen": 1814691840 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045783838383838384, + "loss": 1.2387, + "theoretical_loss": 3.4585749848837133, + "tokens_seen": 1814757376 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578181818181818, + "loss": 1.2531, + "theoretical_loss": 3.45856437123138, + "tokens_seen": 1814822912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577979797979798, + "loss": 1.4417, + "theoretical_loss": 3.4585537580696264, + "tokens_seen": 1814888448 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577777777777778, + "loss": 1.6412, + "theoretical_loss": 3.458543145398414, + "tokens_seen": 1814953984 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045775757575757577, + "loss": 1.6142, + "theoretical_loss": 3.458532533217701, + "tokens_seen": 1815019520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045773737373737373, + "loss": 1.5157, + "theoretical_loss": 3.4585219215274474, + "tokens_seen": 1815085056 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045771717171717174, + "loss": 1.43, + "theoretical_loss": 3.458511310327613, + "tokens_seen": 1815150592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576969696969697, + "loss": 1.2891, + "theoretical_loss": 3.458500699618158, + "tokens_seen": 1815216128 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576767676767677, + "loss": 1.4889, + "theoretical_loss": 3.458490089399041, + "tokens_seen": 1815281664 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045765656565656567, + "loss": 1.5082, + "theoretical_loss": 3.4584794796702223, + "tokens_seen": 1815347200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576363636363636, + "loss": 1.3993, + "theoretical_loss": 3.458468870431661, + "tokens_seen": 1815412736 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045761616161616163, + "loss": 1.4451, + "theoretical_loss": 3.4584582616833175, + "tokens_seen": 1815478272 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045759595959595964, + "loss": 1.3518, + "theoretical_loss": 3.458447653425151, + "tokens_seen": 1815543808 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.34109237790107727, + "objective/train/avg_token_score": 0.014077051542699337, + "objective/train/docs_used": 1026828, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3032065629959106, + "objective/train/original_loss": 4.5359601974487305, + "objective/train/theoretical_loss": 3.4584370456571207, + "objective/train/tokens_used": 174468576, + "objective/train/unlikelihood_loss": 0.057071033865213394, + "theoretical_loss": 3.4584370456571207, + "tokens_seen": 1815609344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575757575757576, + "loss": 1.4086, + "theoretical_loss": 3.4584370456571207, + "tokens_seen": 1815609344 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045755555555555556, + "loss": 1.3255, + "theoretical_loss": 3.4584264383791874, + "tokens_seen": 1815674880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575353535353535, + "loss": 1.5165, + "theoretical_loss": 3.4584158315913096, + "tokens_seen": 1815740416 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575151515151515, + "loss": 1.4522, + "theoretical_loss": 3.458405225293448, + "tokens_seen": 1815805952 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045749494949494954, + "loss": 1.4464, + "theoretical_loss": 3.458394619485561, + "tokens_seen": 1815871488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574747474747475, + "loss": 1.495, + "theoretical_loss": 3.4583840141676094, + "tokens_seen": 1815937024 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045745454545454545, + "loss": 1.6631, + "theoretical_loss": 3.458373409339553, + "tokens_seen": 1816002560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574343434343434, + "loss": 1.4861, + "theoretical_loss": 3.4583628050013506, + "tokens_seen": 1816068096 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574141414141415, + "loss": 1.312, + "theoretical_loss": 3.4583522011529624, + "tokens_seen": 1816133632 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045739393939393943, + "loss": 1.3567, + "theoretical_loss": 3.458341597794348, + "tokens_seen": 1816199168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004573737373737374, + "loss": 1.4321, + "theoretical_loss": 3.4583309949254675, + "tokens_seen": 1816264704 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045735353535353534, + "loss": 1.3804, + "theoretical_loss": 3.4583203925462795, + "tokens_seen": 1816330240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004573333333333333, + "loss": 1.292, + "theoretical_loss": 3.4583097906567453, + "tokens_seen": 1816395776 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045731313131313137, + "loss": 1.3965, + "theoretical_loss": 3.458299189256823, + "tokens_seen": 1816461312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572929292929293, + "loss": 1.532, + "theoretical_loss": 3.458288588346474, + "tokens_seen": 1816526848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572727272727273, + "loss": 1.5313, + "theoretical_loss": 3.4582779879256567, + "tokens_seen": 1816592384 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045725252525252524, + "loss": 1.314, + "theoretical_loss": 3.458267387994331, + "tokens_seen": 1816657920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045723232323232325, + "loss": 1.4601, + "theoretical_loss": 3.458256788552457, + "tokens_seen": 1816723456 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045721212121212126, + "loss": 1.169, + "theoretical_loss": 3.4582461895999947, + "tokens_seen": 1816788992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571919191919192, + "loss": 1.7361, + "theoretical_loss": 3.458235591136903, + "tokens_seen": 1816854528 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045717171717171717, + "loss": 1.4227, + "theoretical_loss": 3.458224993163143, + "tokens_seen": 1816920064 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045715151515151513, + "loss": 1.4055, + "theoretical_loss": 3.458214395678673, + "tokens_seen": 1816985600 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045713131313131314, + "loss": 1.37, + "theoretical_loss": 3.4582037986834537, + "tokens_seen": 1817051136 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045711111111111115, + "loss": 1.6846, + "theoretical_loss": 3.458193202177444, + "tokens_seen": 1817116672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004570909090909091, + "loss": 1.2868, + "theoretical_loss": 3.4581826061606047, + "tokens_seen": 1817182208 + }, + { + "epoch": 0.09, + "objective/train/aligned_token_count": 0.3228861093521118, + "objective/train/avg_token_score": 0.038064464926719666, + "objective/train/docs_used": 1028029, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0878517627716064, + "objective/train/original_loss": 4.760405540466309, + "objective/train/theoretical_loss": 3.458172010632895, + "objective/train/tokens_used": 176106976, + "objective/train/unlikelihood_loss": 0.04580136016011238, + "theoretical_loss": 3.458172010632895, + "tokens_seen": 1817247744 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045707070707070706, + "loss": 1.159, + "theoretical_loss": 3.458172010632895, + "tokens_seen": 1817247744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004570505050505051, + "loss": 1.4579, + "theoretical_loss": 3.458161415594275, + "tokens_seen": 1817313280 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045703030303030303, + "loss": 1.3157, + "theoretical_loss": 3.458150821044704, + "tokens_seen": 1817378816 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045701010101010104, + "loss": 1.324, + "theoretical_loss": 3.458140226984142, + "tokens_seen": 1817444352 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456989898989899, + "loss": 1.2537, + "theoretical_loss": 3.458129633412549, + "tokens_seen": 1817509888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045696969696969696, + "loss": 1.6337, + "theoretical_loss": 3.4581190403298847, + "tokens_seen": 1817575424 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045694949494949497, + "loss": 1.3909, + "theoretical_loss": 3.458108447736109, + "tokens_seen": 1817640960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569292929292929, + "loss": 1.3148, + "theoretical_loss": 3.4580978556311814, + "tokens_seen": 1817706496 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045690909090909094, + "loss": 1.2169, + "theoretical_loss": 3.458087264015062, + "tokens_seen": 1817772032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004568888888888889, + "loss": 1.5658, + "theoretical_loss": 3.45807667288771, + "tokens_seen": 1817837568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004568686868686869, + "loss": 1.5285, + "theoretical_loss": 3.4580660822490863, + "tokens_seen": 1817903104 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045684848484848486, + "loss": 1.2645, + "theoretical_loss": 3.45805549209915, + "tokens_seen": 1817968640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004568282828282828, + "loss": 1.5466, + "theoretical_loss": 3.4580449024378614, + "tokens_seen": 1818034176 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045680808080808083, + "loss": 1.4763, + "theoretical_loss": 3.4580343132651796, + "tokens_seen": 1818099712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567878787878788, + "loss": 1.3013, + "theoretical_loss": 3.458023724581065, + "tokens_seen": 1818165248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567676767676768, + "loss": 1.4691, + "theoretical_loss": 3.4580131363854782, + "tokens_seen": 1818230784 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045674747474747475, + "loss": 1.3829, + "theoretical_loss": 3.458002548678377, + "tokens_seen": 1818296320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567272727272727, + "loss": 1.4288, + "theoretical_loss": 3.457991961459723, + "tokens_seen": 1818361856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567070707070707, + "loss": 1.7387, + "theoretical_loss": 3.4579813747294756, + "tokens_seen": 1818427392 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045668686868686873, + "loss": 1.2347, + "theoretical_loss": 3.4579707884875948, + "tokens_seen": 1818492928 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566666666666667, + "loss": 1.3851, + "theoretical_loss": 3.4579602027340393, + "tokens_seen": 1818558464 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045664646464646465, + "loss": 1.4735, + "theoretical_loss": 3.457949617468771, + "tokens_seen": 1818624000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566262626262626, + "loss": 1.4051, + "theoretical_loss": 3.457939032691748, + "tokens_seen": 1818689536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566060606060606, + "loss": 1.4462, + "theoretical_loss": 3.4579284484029316, + "tokens_seen": 1818755072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565858585858586, + "loss": 1.2984, + "theoretical_loss": 3.45791786460228, + "tokens_seen": 1818820608 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.464809387922287, + "objective/train/avg_token_score": 0.016029013320803642, + "objective/train/docs_used": 1028792, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6073720455169678, + "objective/train/original_loss": 3.9546051025390625, + "objective/train/theoretical_loss": 3.4579072812897547, + "objective/train/tokens_used": 177745376, + "objective/train/unlikelihood_loss": 0.07868336141109467, + "theoretical_loss": 3.4579072812897547, + "tokens_seen": 1818886144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565656565656566, + "loss": 1.5138, + "theoretical_loss": 3.4579072812897547, + "tokens_seen": 1818886144 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045654545454545454, + "loss": 1.4139, + "theoretical_loss": 3.457896698465315, + "tokens_seen": 1818951680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565252525252525, + "loss": 1.5099, + "theoretical_loss": 3.457886116128921, + "tokens_seen": 1819017216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565050505050505, + "loss": 1.3184, + "theoretical_loss": 3.4578755342805323, + "tokens_seen": 1819082752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564848484848485, + "loss": 1.4564, + "theoretical_loss": 3.457864952920109, + "tokens_seen": 1819148288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564646464646465, + "loss": 1.4435, + "theoretical_loss": 3.4578543720476103, + "tokens_seen": 1819213824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045644444444444443, + "loss": 1.6601, + "theoretical_loss": 3.4578437916629974, + "tokens_seen": 1819279360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564242424242424, + "loss": 1.3769, + "theoretical_loss": 3.4578332117662294, + "tokens_seen": 1819344896 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045640404040404045, + "loss": 1.4961, + "theoretical_loss": 3.4578226323572663, + "tokens_seen": 1819410432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563838383838384, + "loss": 1.2399, + "theoretical_loss": 3.4578120534360686, + "tokens_seen": 1819475968 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045636363636363637, + "loss": 1.5018, + "theoretical_loss": 3.4578014750025954, + "tokens_seen": 1819541504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563434343434343, + "loss": 1.4262, + "theoretical_loss": 3.457790897056807, + "tokens_seen": 1819607040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563232323232323, + "loss": 1.4682, + "theoretical_loss": 3.4577803195986636, + "tokens_seen": 1819672576 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045630303030303035, + "loss": 1.5494, + "theoretical_loss": 3.457769742628125, + "tokens_seen": 1819738112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562828282828283, + "loss": 1.4778, + "theoretical_loss": 3.4577591661451517, + "tokens_seen": 1819803648 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045626262626262626, + "loss": 1.459, + "theoretical_loss": 3.457748590149703, + "tokens_seen": 1819869184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562424242424242, + "loss": 1.5946, + "theoretical_loss": 3.457738014641738, + "tokens_seen": 1819934720 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045622222222222223, + "loss": 1.3618, + "theoretical_loss": 3.4577274396212183, + "tokens_seen": 1820000256 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045620202020202024, + "loss": 1.2889, + "theoretical_loss": 3.457716865088103, + "tokens_seen": 1820065792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561818181818182, + "loss": 1.3204, + "theoretical_loss": 3.457706291042353, + "tokens_seen": 1820131328 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045616161616161615, + "loss": 1.4486, + "theoretical_loss": 3.4576957174839267, + "tokens_seen": 1820196864 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561414141414141, + "loss": 1.2579, + "theoretical_loss": 3.4576851444127854, + "tokens_seen": 1820262400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561212121212121, + "loss": 1.2631, + "theoretical_loss": 3.457674571828889, + "tokens_seen": 1820327936 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045610101010101013, + "loss": 1.2376, + "theoretical_loss": 3.457663999732197, + "tokens_seen": 1820393472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560808080808081, + "loss": 1.3863, + "theoretical_loss": 3.4576534281226694, + "tokens_seen": 1820459008 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.5623167157173157, + "objective/train/avg_token_score": 0.014118832536041737, + "objective/train/docs_used": 1029550, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7937812805175781, + "objective/train/original_loss": 3.694382429122925, + "objective/train/theoretical_loss": 3.457642857000267, + "objective/train/tokens_used": 179383776, + "objective/train/unlikelihood_loss": 0.07253962755203247, + "theoretical_loss": 3.457642857000267, + "tokens_seen": 1820524544 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045606060606060605, + "loss": 1.5616, + "theoretical_loss": 3.457642857000267, + "tokens_seen": 1820524544 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045604040404040406, + "loss": 1.4623, + "theoretical_loss": 3.457632286364949, + "tokens_seen": 1820590080 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045602020202020207, + "loss": 1.5794, + "theoretical_loss": 3.4576217162166754, + "tokens_seen": 1820655616 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456, + "loss": 1.4721, + "theoretical_loss": 3.457611146555407, + "tokens_seen": 1820721152 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455979797979798, + "loss": 1.5718, + "theoretical_loss": 3.457600577381103, + "tokens_seen": 1820786688 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045595959595959594, + "loss": 1.3938, + "theoretical_loss": 3.4575900086937237, + "tokens_seen": 1820852224 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045593939393939395, + "loss": 1.5038, + "theoretical_loss": 3.4575794404932294, + "tokens_seen": 1820917760 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045591919191919196, + "loss": 1.4119, + "theoretical_loss": 3.4575688727795804, + "tokens_seen": 1820983296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558989898989899, + "loss": 1.3345, + "theoretical_loss": 3.4575583055527357, + "tokens_seen": 1821048832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558787878787879, + "loss": 1.2874, + "theoretical_loss": 3.4575477388126563, + "tokens_seen": 1821114368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558585858585859, + "loss": 1.1633, + "theoretical_loss": 3.457537172559302, + "tokens_seen": 1821179904 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045583838383838384, + "loss": 1.4255, + "theoretical_loss": 3.4575266067926327, + "tokens_seen": 1821245440 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045581818181818185, + "loss": 1.4251, + "theoretical_loss": 3.4575160415126085, + "tokens_seen": 1821310976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557979797979798, + "loss": 1.4595, + "theoretical_loss": 3.4575054767191897, + "tokens_seen": 1821376512 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045577777777777777, + "loss": 1.4112, + "theoretical_loss": 3.4574949124123364, + "tokens_seen": 1821442048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557575757575758, + "loss": 1.3731, + "theoretical_loss": 3.457484348592008, + "tokens_seen": 1821507584 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045573737373737374, + "loss": 1.6005, + "theoretical_loss": 3.4574737852581654, + "tokens_seen": 1821573120 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045571717171717175, + "loss": 1.4437, + "theoretical_loss": 3.4574632224107686, + "tokens_seen": 1821638656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556969696969697, + "loss": 1.6006, + "theoretical_loss": 3.4574526600497775, + "tokens_seen": 1821704192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556767676767677, + "loss": 1.3622, + "theoretical_loss": 3.457442098175152, + "tokens_seen": 1821769728 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045565656565656567, + "loss": 1.4977, + "theoretical_loss": 3.457431536786853, + "tokens_seen": 1821835264 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045563636363636363, + "loss": 1.5852, + "theoretical_loss": 3.4574209758848395, + "tokens_seen": 1821900800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045561616161616164, + "loss": 1.3857, + "theoretical_loss": 3.457410415469072, + "tokens_seen": 1821966336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555959595959596, + "loss": 1.5078, + "theoretical_loss": 3.4573998555395113, + "tokens_seen": 1822031872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555757575757576, + "loss": 1.4179, + "theoretical_loss": 3.4573892960961166, + "tokens_seen": 1822097408 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.4111681282520294, + "objective/train/avg_token_score": 0.017115775495767593, + "objective/train/docs_used": 1030259, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2357131242752075, + "objective/train/original_loss": 3.788912296295166, + "objective/train/theoretical_loss": 3.457378737138849, + "objective/train/tokens_used": 181022176, + "objective/train/unlikelihood_loss": 0.0951184332370758, + "theoretical_loss": 3.457378737138849, + "tokens_seen": 1822162944 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045555555555555556, + "loss": 1.2408, + "theoretical_loss": 3.457378737138849, + "tokens_seen": 1822162944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555353535353535, + "loss": 1.3708, + "theoretical_loss": 3.4573681786676675, + "tokens_seen": 1822228480 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045551515151515153, + "loss": 1.5096, + "theoretical_loss": 3.457357620682533, + "tokens_seen": 1822294016 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045549494949494954, + "loss": 1.3569, + "theoretical_loss": 3.4573470631834056, + "tokens_seen": 1822359552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554747474747475, + "loss": 1.3497, + "theoretical_loss": 3.457336506170245, + "tokens_seen": 1822425088 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045545454545454546, + "loss": 1.5165, + "theoretical_loss": 3.457325949643012, + "tokens_seen": 1822490624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554343434343434, + "loss": 1.5752, + "theoretical_loss": 3.4573153936016663, + "tokens_seen": 1822556160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554141414141414, + "loss": 1.2614, + "theoretical_loss": 3.4573048380461686, + "tokens_seen": 1822621696 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045539393939393944, + "loss": 1.4106, + "theoretical_loss": 3.4572942829764783, + "tokens_seen": 1822687232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553737373737374, + "loss": 1.4145, + "theoretical_loss": 3.4572837283925564, + "tokens_seen": 1822752768 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045535353535353535, + "loss": 1.4457, + "theoretical_loss": 3.457273174294362, + "tokens_seen": 1822818304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553333333333333, + "loss": 1.418, + "theoretical_loss": 3.457262620681856, + "tokens_seen": 1822883840 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045531313131313137, + "loss": 1.3548, + "theoretical_loss": 3.4572520675549985, + "tokens_seen": 1822949376 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045529292929292933, + "loss": 1.4232, + "theoretical_loss": 3.45724151491375, + "tokens_seen": 1823014912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552727272727273, + "loss": 1.5722, + "theoretical_loss": 3.4572309627580706, + "tokens_seen": 1823080448 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045525252525252524, + "loss": 1.6458, + "theoretical_loss": 3.4572204110879197, + "tokens_seen": 1823145984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552323232323232, + "loss": 1.352, + "theoretical_loss": 3.4572098599032586, + "tokens_seen": 1823211520 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045521212121212126, + "loss": 1.6786, + "theoretical_loss": 3.4571993092040465, + "tokens_seen": 1823277056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551919191919192, + "loss": 1.6506, + "theoretical_loss": 3.4571887589902444, + "tokens_seen": 1823342592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551717171717172, + "loss": 1.2894, + "theoretical_loss": 3.457178209261812, + "tokens_seen": 1823408128 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045515151515151513, + "loss": 1.3039, + "theoretical_loss": 3.457167660018711, + "tokens_seen": 1823473664 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551313131313131, + "loss": 1.1383, + "theoretical_loss": 3.4571571112608988, + "tokens_seen": 1823539200 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045511111111111116, + "loss": 1.3979, + "theoretical_loss": 3.4571465629883384, + "tokens_seen": 1823604736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550909090909091, + "loss": 1.3313, + "theoretical_loss": 3.4571360152009882, + "tokens_seen": 1823670272 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045507070707070707, + "loss": 1.5223, + "theoretical_loss": 3.4571254678988095, + "tokens_seen": 1823735808 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.38483625650405884, + "objective/train/avg_token_score": 0.019340477883815765, + "objective/train/docs_used": 1031478, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2517280578613281, + "objective/train/original_loss": 4.015500068664551, + "objective/train/theoretical_loss": 3.4571149210817618, + "objective/train/tokens_used": 182660576, + "objective/train/unlikelihood_loss": 0.09212837368249893, + "theoretical_loss": 3.4571149210817618, + "tokens_seen": 1823801344 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045505050505050503, + "loss": 1.3643, + "theoretical_loss": 3.4571149210817618, + "tokens_seen": 1823801344 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045503030303030304, + "loss": 1.4704, + "theoretical_loss": 3.4571043747498065, + "tokens_seen": 1823866880 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045501010101010105, + "loss": 1.5384, + "theoretical_loss": 3.4570938289029023, + "tokens_seen": 1823932416 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454989898989899, + "loss": 1.518, + "theoretical_loss": 3.457083283541011, + "tokens_seen": 1823997952 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045496969696969696, + "loss": 1.1422, + "theoretical_loss": 3.4570727386640914, + "tokens_seen": 1824063488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549494949494949, + "loss": 1.3168, + "theoretical_loss": 3.4570621942721047, + "tokens_seen": 1824129024 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045492929292929293, + "loss": 1.3456, + "theoretical_loss": 3.4570516503650115, + "tokens_seen": 1824194560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045490909090909094, + "loss": 1.4976, + "theoretical_loss": 3.4570411069427713, + "tokens_seen": 1824260096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548888888888889, + "loss": 1.4185, + "theoretical_loss": 3.4570305640053443, + "tokens_seen": 1824325632 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045486868686868686, + "loss": 1.3554, + "theoretical_loss": 3.457020021552691, + "tokens_seen": 1824391168 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045484848484848487, + "loss": 1.4911, + "theoretical_loss": 3.4570094795847726, + "tokens_seen": 1824456704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548282828282828, + "loss": 1.3135, + "theoretical_loss": 3.456998938101548, + "tokens_seen": 1824522240 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045480808080808083, + "loss": 1.0882, + "theoretical_loss": 3.456988397102979, + "tokens_seen": 1824587776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547878787878788, + "loss": 1.3685, + "theoretical_loss": 3.4569778565890243, + "tokens_seen": 1824653312 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045476767676767675, + "loss": 1.4268, + "theoretical_loss": 3.456967316559645, + "tokens_seen": 1824718848 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045474747474747476, + "loss": 1.5668, + "theoretical_loss": 3.4569567770148018, + "tokens_seen": 1824784384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547272727272727, + "loss": 1.6636, + "theoretical_loss": 3.4569462379544547, + "tokens_seen": 1824849920 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045470707070707073, + "loss": 1.4786, + "theoretical_loss": 3.4569356993785636, + "tokens_seen": 1824915456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546868686868687, + "loss": 1.3914, + "theoretical_loss": 3.456925161287089, + "tokens_seen": 1824980992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546666666666667, + "loss": 1.2798, + "theoretical_loss": 3.456914623679992, + "tokens_seen": 1825046528 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045464646464646465, + "loss": 1.4608, + "theoretical_loss": 3.4569040865572322, + "tokens_seen": 1825112064 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546262626262626, + "loss": 1.6782, + "theoretical_loss": 3.4568935499187696, + "tokens_seen": 1825177600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546060606060606, + "loss": 1.4233, + "theoretical_loss": 3.456883013764566, + "tokens_seen": 1825243136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545858585858586, + "loss": 1.5035, + "theoretical_loss": 3.4568724780945796, + "tokens_seen": 1825308672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545656565656566, + "loss": 1.4682, + "theoretical_loss": 3.456861942908773, + "tokens_seen": 1825374208 + }, + { + "debugging/Self-BLEU-5": 0.383412367211192, + "debugging/distinct-1-grams": 0.7526443544193546, + "debugging/distinct-2-grams": 0.9328595808568966, + "debugging/entropy-1-grams": 5.657827037788948, + "debugging/entropy-2-grams": 6.397750202501998, + "debugging/length": 484.77777777777777, + "debugging/num_segments": 9, + "epoch": 0.1, + "objective/train/aligned_token_count": 0.5272482633590698, + "objective/train/avg_token_score": 0.014473862946033478, + "objective/train/docs_used": 1032081, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6364436149597168, + "objective/train/original_loss": 3.784334897994995, + "objective/train/theoretical_loss": 3.456851408207105, + "objective/train/tokens_used": 184298976, + "objective/train/unlikelihood_loss": 0.08270136266946793, + "theoretical_loss": 3.456851408207105, + "tokens_seen": 1825439744 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045454545454545455, + "loss": 1.618, + "theoretical_loss": 3.456851408207105, + "tokens_seen": 1825439744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545252525252525, + "loss": 1.2274, + "theoretical_loss": 3.456840873989537, + "tokens_seen": 1825505280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545050505050505, + "loss": 1.3231, + "theoretical_loss": 3.4568303402560288, + "tokens_seen": 1825570816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544848484848485, + "loss": 1.3608, + "theoretical_loss": 3.4568198070065406, + "tokens_seen": 1825636352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544646464646465, + "loss": 1.4591, + "theoretical_loss": 3.4568092742410332, + "tokens_seen": 1825701888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045444444444444444, + "loss": 1.2171, + "theoretical_loss": 3.456798741959467, + "tokens_seen": 1825767424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544242424242424, + "loss": 1.4365, + "theoretical_loss": 3.456788210161802, + "tokens_seen": 1825832960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544040404040404, + "loss": 1.4114, + "theoretical_loss": 3.456777678847999, + "tokens_seen": 1825898496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543838383838384, + "loss": 1.3937, + "theoretical_loss": 3.4567671480180184, + "tokens_seen": 1825964032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543636363636364, + "loss": 1.4272, + "theoretical_loss": 3.4567566176718203, + "tokens_seen": 1826029568 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045434343434343433, + "loss": 1.5479, + "theoretical_loss": 3.456746087809365, + "tokens_seen": 1826095104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543232323232323, + "loss": 1.1661, + "theoretical_loss": 3.456735558430614, + "tokens_seen": 1826160640 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045430303030303035, + "loss": 1.1125, + "theoretical_loss": 3.4567250295355265, + "tokens_seen": 1826226176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542828282828283, + "loss": 1.4769, + "theoretical_loss": 3.456714501124063, + "tokens_seen": 1826291712 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045426262626262627, + "loss": 1.3936, + "theoretical_loss": 3.4567039731961846, + "tokens_seen": 1826357248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542424242424242, + "loss": 1.2966, + "theoretical_loss": 3.4566934457518514, + "tokens_seen": 1826422784 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045422222222222223, + "loss": 1.341, + "theoretical_loss": 3.456682918791024, + "tokens_seen": 1826488320 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045420202020202025, + "loss": 1.4049, + "theoretical_loss": 3.4566723923136626, + "tokens_seen": 1826553856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541818181818182, + "loss": 1.4241, + "theoretical_loss": 3.4566618663197275, + "tokens_seen": 1826619392 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045416161616161616, + "loss": 1.3875, + "theoretical_loss": 3.45665134080918, + "tokens_seen": 1826684928 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541414141414141, + "loss": 1.5876, + "theoretical_loss": 3.4566408157819795, + "tokens_seen": 1826750464 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541212121212122, + "loss": 1.3572, + "theoretical_loss": 3.456630291238087, + "tokens_seen": 1826816000 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045410101010101014, + "loss": 1.5581, + "theoretical_loss": 3.4566197671774628, + "tokens_seen": 1826881536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540808080808081, + "loss": 1.4737, + "theoretical_loss": 3.456609243600068, + "tokens_seen": 1826947072 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045406060606060605, + "loss": 1.428, + "theoretical_loss": 3.456598720505862, + "tokens_seen": 1827012608 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.3259408473968506, + "objective/train/avg_token_score": 0.02392873726785183, + "objective/train/docs_used": 1033277, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0870252847671509, + "objective/train/original_loss": 3.9227612018585205, + "objective/train/theoretical_loss": 3.4565881978948063, + "objective/train/tokens_used": 185937376, + "objective/train/unlikelihood_loss": 0.08083102852106094, + "theoretical_loss": 3.4565881978948063, + "tokens_seen": 1827078144 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454040404040404, + "loss": 1.1697, + "theoretical_loss": 3.4565881978948063, + "tokens_seen": 1827078144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540202020202021, + "loss": 1.3098, + "theoretical_loss": 3.4565776757668605, + "tokens_seen": 1827143680 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045400000000000003, + "loss": 1.3512, + "theoretical_loss": 3.456567154121986, + "tokens_seen": 1827209216 + }, + { + "epoch": 0.1, + "learning_rate": 0.000453979797979798, + "loss": 1.438, + "theoretical_loss": 3.4565566329601425, + "tokens_seen": 1827274752 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045395959595959594, + "loss": 1.6695, + "theoretical_loss": 3.4565461122812904, + "tokens_seen": 1827340288 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045393939393939396, + "loss": 1.1057, + "theoretical_loss": 3.4565355920853915, + "tokens_seen": 1827405824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045391919191919197, + "loss": 1.5515, + "theoretical_loss": 3.456525072372405, + "tokens_seen": 1827471360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538989898989899, + "loss": 1.2735, + "theoretical_loss": 3.456514553142292, + "tokens_seen": 1827536896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538787878787879, + "loss": 1.3733, + "theoretical_loss": 3.456504034395013, + "tokens_seen": 1827602432 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045385858585858584, + "loss": 1.3649, + "theoretical_loss": 3.4564935161305277, + "tokens_seen": 1827667968 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045383838383838385, + "loss": 1.7118, + "theoretical_loss": 3.4564829983487977, + "tokens_seen": 1827733504 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045381818181818186, + "loss": 1.3295, + "theoretical_loss": 3.4564724810497838, + "tokens_seen": 1827799040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537979797979798, + "loss": 1.2664, + "theoretical_loss": 3.456461964233445, + "tokens_seen": 1827864576 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537777777777778, + "loss": 1.5361, + "theoretical_loss": 3.4564514478997435, + "tokens_seen": 1827930112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537575757575758, + "loss": 1.5431, + "theoretical_loss": 3.456440932048639, + "tokens_seen": 1827995648 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045373737373737374, + "loss": 1.377, + "theoretical_loss": 3.4564304166800914, + "tokens_seen": 1828061184 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045371717171717175, + "loss": 1.3454, + "theoretical_loss": 3.456419901794063, + "tokens_seen": 1828126720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536969696969697, + "loss": 1.4446, + "theoretical_loss": 3.4564093873905124, + "tokens_seen": 1828192256 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045367676767676767, + "loss": 1.4514, + "theoretical_loss": 3.4563988734694018, + "tokens_seen": 1828257792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536565656565657, + "loss": 1.3498, + "theoretical_loss": 3.456388360030691, + "tokens_seen": 1828323328 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045363636363636363, + "loss": 1.4994, + "theoretical_loss": 3.4563778470743403, + "tokens_seen": 1828388864 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045361616161616164, + "loss": 1.1171, + "theoretical_loss": 3.456367334600311, + "tokens_seen": 1828454400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535959595959596, + "loss": 1.37, + "theoretical_loss": 3.4563568226085635, + "tokens_seen": 1828519936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535757575757576, + "loss": 1.4101, + "theoretical_loss": 3.456346311099058, + "tokens_seen": 1828585472 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045355555555555557, + "loss": 1.2472, + "theoretical_loss": 3.4563358000717552, + "tokens_seen": 1828651008 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.3474462330341339, + "objective/train/avg_token_score": 0.01256563700735569, + "objective/train/docs_used": 1034021, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1423338651657104, + "objective/train/original_loss": 4.592188835144043, + "objective/train/theoretical_loss": 3.456325289526616, + "objective/train/tokens_used": 187575776, + "objective/train/unlikelihood_loss": 0.06064225360751152, + "theoretical_loss": 3.456325289526616, + "tokens_seen": 1828716544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535353535353535, + "loss": 1.4799, + "theoretical_loss": 3.456325289526616, + "tokens_seen": 1828716544 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045351515151515154, + "loss": 1.2678, + "theoretical_loss": 3.456314779463601, + "tokens_seen": 1828782080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534949494949495, + "loss": 1.3876, + "theoretical_loss": 3.4563042698826707, + "tokens_seen": 1828847616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534747474747475, + "loss": 1.4213, + "theoretical_loss": 3.456293760783786, + "tokens_seen": 1828913152 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045345454545454546, + "loss": 1.596, + "theoretical_loss": 3.456283252166906, + "tokens_seen": 1828978688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534343434343434, + "loss": 1.253, + "theoretical_loss": 3.4562727440319936, + "tokens_seen": 1829044224 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045341414141414143, + "loss": 1.4315, + "theoretical_loss": 3.4562622363790076, + "tokens_seen": 1829109760 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045339393939393944, + "loss": 1.1965, + "theoretical_loss": 3.45625172920791, + "tokens_seen": 1829175296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533737373737374, + "loss": 1.3171, + "theoretical_loss": 3.45624122251866, + "tokens_seen": 1829240832 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045335353535353535, + "loss": 1.2845, + "theoretical_loss": 3.45623071631122, + "tokens_seen": 1829306368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533333333333333, + "loss": 1.6198, + "theoretical_loss": 3.456220210585549, + "tokens_seen": 1829371904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533131313131313, + "loss": 1.268, + "theoretical_loss": 3.4562097053416085, + "tokens_seen": 1829437440 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045329292929292933, + "loss": 1.3782, + "theoretical_loss": 3.456199200579359, + "tokens_seen": 1829502976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532727272727273, + "loss": 1.4489, + "theoretical_loss": 3.456188696298761, + "tokens_seen": 1829568512 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045325252525252525, + "loss": 1.3057, + "theoretical_loss": 3.4561781924997756, + "tokens_seen": 1829634048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532323232323232, + "loss": 1.5815, + "theoretical_loss": 3.456167689182363, + "tokens_seen": 1829699584 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532121212121212, + "loss": 1.5887, + "theoretical_loss": 3.456157186346484, + "tokens_seen": 1829765120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531919191919192, + "loss": 1.5131, + "theoretical_loss": 3.4561466839921, + "tokens_seen": 1829830656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531717171717172, + "loss": 1.6467, + "theoretical_loss": 3.4561361821191703, + "tokens_seen": 1829896192 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045315151515151514, + "loss": 1.54, + "theoretical_loss": 3.456125680727656, + "tokens_seen": 1829961728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531313131313131, + "loss": 1.6346, + "theoretical_loss": 3.4561151798175187, + "tokens_seen": 1830027264 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045311111111111116, + "loss": 1.2857, + "theoretical_loss": 3.4561046793887185, + "tokens_seen": 1830092800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530909090909091, + "loss": 1.1951, + "theoretical_loss": 3.456094179441216, + "tokens_seen": 1830158336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530707070707071, + "loss": 1.2465, + "theoretical_loss": 3.456083679974972, + "tokens_seen": 1830223872 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045305050505050503, + "loss": 1.3245, + "theoretical_loss": 3.456073180989947, + "tokens_seen": 1830289408 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.4240591526031494, + "objective/train/avg_token_score": 0.011298497207462788, + "objective/train/docs_used": 1035168, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4181580543518066, + "objective/train/original_loss": 4.181857585906982, + "objective/train/theoretical_loss": 3.456062682486102, + "objective/train/tokens_used": 189214176, + "objective/train/unlikelihood_loss": 0.05961797386407852, + "theoretical_loss": 3.456062682486102, + "tokens_seen": 1830354944 + }, + { + "epoch": 0.1, + "learning_rate": 0.000453030303030303, + "loss": 1.5355, + "theoretical_loss": 3.456062682486102, + "tokens_seen": 1830354944 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045301010101010106, + "loss": 1.466, + "theoretical_loss": 3.4560521844633976, + "tokens_seen": 1830420480 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452989898989899, + "loss": 1.5066, + "theoretical_loss": 3.456041686921795, + "tokens_seen": 1830486016 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045296969696969697, + "loss": 1.3351, + "theoretical_loss": 3.456031189861254, + "tokens_seen": 1830551552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529494949494949, + "loss": 1.2997, + "theoretical_loss": 3.4560206932817357, + "tokens_seen": 1830617088 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045292929292929294, + "loss": 1.5116, + "theoretical_loss": 3.4560101971832013, + "tokens_seen": 1830682624 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045290909090909095, + "loss": 1.0942, + "theoretical_loss": 3.455999701565611, + "tokens_seen": 1830748160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528888888888889, + "loss": 1.3721, + "theoretical_loss": 3.455989206428926, + "tokens_seen": 1830813696 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045286868686868686, + "loss": 1.2505, + "theoretical_loss": 3.455978711773107, + "tokens_seen": 1830879232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528484848484848, + "loss": 1.6109, + "theoretical_loss": 3.455968217598114, + "tokens_seen": 1830944768 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045282828282828283, + "loss": 1.3838, + "theoretical_loss": 3.455957723903909, + "tokens_seen": 1831010304 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045280808080808084, + "loss": 1.2037, + "theoretical_loss": 3.4559472306904517, + "tokens_seen": 1831075840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004527878787878788, + "loss": 1.3973, + "theoretical_loss": 3.4559367379577033, + "tokens_seen": 1831141376 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045276767676767675, + "loss": 1.4359, + "theoretical_loss": 3.4559262457056246, + "tokens_seen": 1831206912 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045274747474747477, + "loss": 1.6251, + "theoretical_loss": 3.4559157539341765, + "tokens_seen": 1831272448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004527272727272727, + "loss": 1.4879, + "theoretical_loss": 3.455905262643319, + "tokens_seen": 1831337984 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045270707070707073, + "loss": 1.7081, + "theoretical_loss": 3.4558947718330137, + "tokens_seen": 1831403520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526868686868687, + "loss": 1.4078, + "theoretical_loss": 3.4558842815032214, + "tokens_seen": 1831469056 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045266666666666665, + "loss": 1.5619, + "theoretical_loss": 3.455873791653903, + "tokens_seen": 1831534592 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045264646464646466, + "loss": 1.4385, + "theoretical_loss": 3.4558633022850183, + "tokens_seen": 1831600128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526262626262626, + "loss": 1.1827, + "theoretical_loss": 3.455852813396529, + "tokens_seen": 1831665664 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526060606060606, + "loss": 1.5327, + "theoretical_loss": 3.455842324988396, + "tokens_seen": 1831731200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525858585858586, + "loss": 1.6505, + "theoretical_loss": 3.455831837060579, + "tokens_seen": 1831796736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525656565656566, + "loss": 1.2537, + "theoretical_loss": 3.45582134961304, + "tokens_seen": 1831862272 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045254545454545455, + "loss": 1.2179, + "theoretical_loss": 3.4558108626457402, + "tokens_seen": 1831927808 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.2812194526195526, + "objective/train/avg_token_score": 0.04614793136715889, + "objective/train/docs_used": 1035908, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9016737937927246, + "objective/train/original_loss": 4.561851978302002, + "objective/train/theoretical_loss": 3.455800376158639, + "objective/train/tokens_used": 190852576, + "objective/train/unlikelihood_loss": 0.06523799896240234, + "theoretical_loss": 3.455800376158639, + "tokens_seen": 1831993344 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045252525252525256, + "loss": 1.376, + "theoretical_loss": 3.455800376158639, + "tokens_seen": 1831993344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525050505050505, + "loss": 1.4152, + "theoretical_loss": 3.4557898901516984, + "tokens_seen": 1832058880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524848484848485, + "loss": 1.565, + "theoretical_loss": 3.455779404624878, + "tokens_seen": 1832124416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524646464646465, + "loss": 1.4454, + "theoretical_loss": 3.45576891957814, + "tokens_seen": 1832189952 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045244444444444444, + "loss": 1.6646, + "theoretical_loss": 3.4557584350114445, + "tokens_seen": 1832255488 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045242424242424245, + "loss": 1.2104, + "theoretical_loss": 3.4557479509247524, + "tokens_seen": 1832321024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524040404040404, + "loss": 1.6734, + "theoretical_loss": 3.455737467318025, + "tokens_seen": 1832386560 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523838383838384, + "loss": 1.4283, + "theoretical_loss": 3.4557269841912226, + "tokens_seen": 1832452096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523636363636364, + "loss": 1.5698, + "theoretical_loss": 3.4557165015443063, + "tokens_seen": 1832517632 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045234343434343434, + "loss": 1.1731, + "theoretical_loss": 3.455706019377237, + "tokens_seen": 1832583168 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045232323232323235, + "loss": 1.2459, + "theoretical_loss": 3.4556955376899756, + "tokens_seen": 1832648704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523030303030303, + "loss": 1.3951, + "theoretical_loss": 3.4556850564824826, + "tokens_seen": 1832714240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004522828282828283, + "loss": 1.247, + "theoretical_loss": 3.4556745757547196, + "tokens_seen": 1832779776 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045226262626262627, + "loss": 1.4864, + "theoretical_loss": 3.455664095506647, + "tokens_seen": 1832845312 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045224242424242423, + "loss": 1.4557, + "theoretical_loss": 3.4556536157382256, + "tokens_seen": 1832910848 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045222222222222224, + "loss": 1.4369, + "theoretical_loss": 3.455643136449417, + "tokens_seen": 1832976384 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045220202020202025, + "loss": 1.4263, + "theoretical_loss": 3.455632657640181, + "tokens_seen": 1833041920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521818181818182, + "loss": 1.458, + "theoretical_loss": 3.4556221793104793, + "tokens_seen": 1833107456 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045216161616161616, + "loss": 1.6213, + "theoretical_loss": 3.4556117014602727, + "tokens_seen": 1833172992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521414141414141, + "loss": 1.3254, + "theoretical_loss": 3.455601224089522, + "tokens_seen": 1833238528 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045212121212121213, + "loss": 1.5437, + "theoretical_loss": 3.4555907471981886, + "tokens_seen": 1833304064 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045210101010101014, + "loss": 1.3216, + "theoretical_loss": 3.455580270786232, + "tokens_seen": 1833369600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520808080808081, + "loss": 1.459, + "theoretical_loss": 3.455569794853615, + "tokens_seen": 1833435136 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045206060606060606, + "loss": 1.4695, + "theoretical_loss": 3.4555593194002974, + "tokens_seen": 1833500672 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452040404040404, + "loss": 1.392, + "theoretical_loss": 3.4555488444262403, + "tokens_seen": 1833566208 + }, + { + "epoch": 0.1, + "objective/train/aligned_token_count": 0.48057183623313904, + "objective/train/avg_token_score": 0.015353728085756302, + "objective/train/docs_used": 1036404, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4945865869522095, + "objective/train/original_loss": 4.170429706573486, + "objective/train/theoretical_loss": 3.4555383699314044, + "objective/train/tokens_used": 192490976, + "objective/train/unlikelihood_loss": 0.0738421231508255, + "theoretical_loss": 3.4555383699314044, + "tokens_seen": 1833631744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520202020202021, + "loss": 1.5297, + "theoretical_loss": 3.4555383699314044, + "tokens_seen": 1833631744 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045200000000000004, + "loss": 1.5953, + "theoretical_loss": 3.4555278959157514, + "tokens_seen": 1833697280 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451979797979798, + "loss": 1.4807, + "theoretical_loss": 3.455517422379242, + "tokens_seen": 1833762816 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045195959595959595, + "loss": 1.4942, + "theoretical_loss": 3.455506949321837, + "tokens_seen": 1833828352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519393939393939, + "loss": 1.3145, + "theoretical_loss": 3.455496476743497, + "tokens_seen": 1833893888 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045191919191919197, + "loss": 1.5973, + "theoretical_loss": 3.4554860046441833, + "tokens_seen": 1833959424 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045189898989898993, + "loss": 1.3642, + "theoretical_loss": 3.455475533023857, + "tokens_seen": 1834024960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518787878787879, + "loss": 1.2289, + "theoretical_loss": 3.455465061882479, + "tokens_seen": 1834090496 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045185858585858584, + "loss": 1.5492, + "theoretical_loss": 3.4554545912200103, + "tokens_seen": 1834156032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518383838383838, + "loss": 1.2268, + "theoretical_loss": 3.455444121036412, + "tokens_seen": 1834221568 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045181818181818186, + "loss": 1.3147, + "theoretical_loss": 3.455433651331645, + "tokens_seen": 1834287104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004517979797979798, + "loss": 1.4772, + "theoretical_loss": 3.45542318210567, + "tokens_seen": 1834352640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004517777777777778, + "loss": 1.4504, + "theoretical_loss": 3.455412713358448, + "tokens_seen": 1834418176 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045175757575757574, + "loss": 1.5777, + "theoretical_loss": 3.45540224508994, + "tokens_seen": 1834483712 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045173737373737375, + "loss": 1.4718, + "theoretical_loss": 3.455391777300108, + "tokens_seen": 1834549248 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045171717171717176, + "loss": 1.5121, + "theoretical_loss": 3.455381309988912, + "tokens_seen": 1834614784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516969696969697, + "loss": 1.544, + "theoretical_loss": 3.455370843156313, + "tokens_seen": 1834680320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045167676767676767, + "loss": 1.6024, + "theoretical_loss": 3.4553603768022727, + "tokens_seen": 1834745856 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045165656565656563, + "loss": 1.4055, + "theoretical_loss": 3.4553499109267514, + "tokens_seen": 1834811392 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045163636363636364, + "loss": 1.3813, + "theoretical_loss": 3.4553394455297104, + "tokens_seen": 1834876928 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045161616161616165, + "loss": 1.4046, + "theoretical_loss": 3.455328980611111, + "tokens_seen": 1834942464 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515959595959596, + "loss": 1.5633, + "theoretical_loss": 3.455318516170914, + "tokens_seen": 1835008000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045157575757575756, + "loss": 1.2908, + "theoretical_loss": 3.4553080522090807, + "tokens_seen": 1835073536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515555555555556, + "loss": 1.2658, + "theoretical_loss": 3.4552975887255717, + "tokens_seen": 1835139072 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045153535353535353, + "loss": 1.0296, + "theoretical_loss": 3.455287125720348, + "tokens_seen": 1835204608 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.46584799885749817, + "objective/train/avg_token_score": 0.026970431208610535, + "objective/train/docs_used": 1037817, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6634101867675781, + "objective/train/original_loss": 4.356515884399414, + "objective/train/theoretical_loss": 3.455276663193372, + "objective/train/tokens_used": 194129376, + "objective/train/unlikelihood_loss": 0.06059441342949867, + "theoretical_loss": 3.455276663193372, + "tokens_seen": 1835270144 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045151515151515154, + "loss": 1.435, + "theoretical_loss": 3.455276663193372, + "tokens_seen": 1835270144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004514949494949495, + "loss": 1.6169, + "theoretical_loss": 3.4552662011446023, + "tokens_seen": 1835335680 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045147474747474746, + "loss": 1.4235, + "theoretical_loss": 3.4552557395740022, + "tokens_seen": 1835401216 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045145454545454547, + "loss": 1.4122, + "theoretical_loss": 3.4552452784815317, + "tokens_seen": 1835466752 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004514343434343434, + "loss": 1.5246, + "theoretical_loss": 3.455234817867152, + "tokens_seen": 1835532288 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045141414141414144, + "loss": 1.4689, + "theoretical_loss": 3.455224357730825, + "tokens_seen": 1835597824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513939393939394, + "loss": 1.2776, + "theoretical_loss": 3.4552138980725102, + "tokens_seen": 1835663360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513737373737374, + "loss": 1.3385, + "theoretical_loss": 3.4552034388921697, + "tokens_seen": 1835728896 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045135353535353536, + "loss": 1.3866, + "theoretical_loss": 3.4551929801897647, + "tokens_seen": 1835794432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513333333333333, + "loss": 1.5819, + "theoretical_loss": 3.455182521965256, + "tokens_seen": 1835859968 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045131313131313133, + "loss": 1.3853, + "theoretical_loss": 3.455172064218605, + "tokens_seen": 1835925504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512929292929293, + "loss": 1.3696, + "theoretical_loss": 3.4551616069497726, + "tokens_seen": 1835991040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512727272727273, + "loss": 1.2499, + "theoretical_loss": 3.4551511501587195, + "tokens_seen": 1836056576 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045125252525252525, + "loss": 1.5628, + "theoretical_loss": 3.4551406938454075, + "tokens_seen": 1836122112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512323232323232, + "loss": 1.6587, + "theoretical_loss": 3.4551302380097972, + "tokens_seen": 1836187648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512121212121212, + "loss": 1.4389, + "theoretical_loss": 3.4551197826518507, + "tokens_seen": 1836253184 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045119191919191923, + "loss": 1.3724, + "theoretical_loss": 3.4551093277715275, + "tokens_seen": 1836318720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511717171717172, + "loss": 1.3688, + "theoretical_loss": 3.45509887336879, + "tokens_seen": 1836384256 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045115151515151515, + "loss": 1.4629, + "theoretical_loss": 3.455088419443599, + "tokens_seen": 1836449792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511313131313131, + "loss": 1.4199, + "theoretical_loss": 3.4550779659959154, + "tokens_seen": 1836515328 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511111111111111, + "loss": 1.4308, + "theoretical_loss": 3.455067513025701, + "tokens_seen": 1836580864 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510909090909091, + "loss": 1.4939, + "theoretical_loss": 3.4550570605329165, + "tokens_seen": 1836646400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510707070707071, + "loss": 1.5078, + "theoretical_loss": 3.455046608517523, + "tokens_seen": 1836711936 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045105050505050504, + "loss": 1.5747, + "theoretical_loss": 3.4550361569794816, + "tokens_seen": 1836777472 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451030303030303, + "loss": 1.4927, + "theoretical_loss": 3.4550257059187537, + "tokens_seen": 1836843008 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.5, + "objective/train/avg_token_score": 0.013788322918117046, + "objective/train/docs_used": 1038484, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6846576929092407, + "objective/train/original_loss": 3.9736297130584717, + "objective/train/theoretical_loss": 3.4550152553353, + "objective/train/tokens_used": 195767776, + "objective/train/unlikelihood_loss": 0.05097836256027222, + "theoretical_loss": 3.4550152553353, + "tokens_seen": 1836908544 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045101010101010106, + "loss": 1.3611, + "theoretical_loss": 3.4550152553353, + "tokens_seen": 1836908544 + }, + { + "epoch": 0.11, + "learning_rate": 0.000450989898989899, + "loss": 1.3674, + "theoretical_loss": 3.4550048052290823, + "tokens_seen": 1836974080 + }, + { + "epoch": 0.11, + "learning_rate": 0.000450969696969697, + "loss": 1.3659, + "theoretical_loss": 3.454994355600062, + "tokens_seen": 1837039616 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045094949494949493, + "loss": 1.2901, + "theoretical_loss": 3.4549839064481995, + "tokens_seen": 1837105152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509292929292929, + "loss": 1.4305, + "theoretical_loss": 3.4549734577734563, + "tokens_seen": 1837170688 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045090909090909095, + "loss": 1.45, + "theoretical_loss": 3.4549630095757937, + "tokens_seen": 1837236224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508888888888889, + "loss": 1.379, + "theoretical_loss": 3.4549525618551726, + "tokens_seen": 1837301760 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045086868686868687, + "loss": 1.6066, + "theoretical_loss": 3.454942114611555, + "tokens_seen": 1837367296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508484848484848, + "loss": 1.4182, + "theoretical_loss": 3.454931667844901, + "tokens_seen": 1837432832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508282828282829, + "loss": 1.3809, + "theoretical_loss": 3.4549212215551726, + "tokens_seen": 1837498368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045080808080808085, + "loss": 1.4271, + "theoretical_loss": 3.4549107757423307, + "tokens_seen": 1837563904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507878787878788, + "loss": 1.4629, + "theoretical_loss": 3.4549003304063364, + "tokens_seen": 1837629440 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045076767676767676, + "loss": 1.6347, + "theoretical_loss": 3.4548898855471517, + "tokens_seen": 1837694976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507474747474747, + "loss": 1.4147, + "theoretical_loss": 3.4548794411647368, + "tokens_seen": 1837760512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507272727272728, + "loss": 1.561, + "theoretical_loss": 3.4548689972590534, + "tokens_seen": 1837826048 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045070707070707074, + "loss": 1.7009, + "theoretical_loss": 3.4548585538300625, + "tokens_seen": 1837891584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506868686868687, + "loss": 1.5077, + "theoretical_loss": 3.454848110877726, + "tokens_seen": 1837957120 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045066666666666665, + "loss": 1.3909, + "theoretical_loss": 3.4548376684020043, + "tokens_seen": 1838022656 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045064646464646466, + "loss": 1.4584, + "theoretical_loss": 3.45482722640286, + "tokens_seen": 1838088192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506262626262627, + "loss": 1.5085, + "theoretical_loss": 3.4548167848802525, + "tokens_seen": 1838153728 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045060606060606063, + "loss": 1.4115, + "theoretical_loss": 3.454806343834144, + "tokens_seen": 1838219264 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505858585858586, + "loss": 1.4466, + "theoretical_loss": 3.454795903264496, + "tokens_seen": 1838284800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045056565656565655, + "loss": 1.7046, + "theoretical_loss": 3.4547854631712696, + "tokens_seen": 1838350336 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045054545454545456, + "loss": 1.6425, + "theoretical_loss": 3.454775023554426, + "tokens_seen": 1838415872 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045052525252525257, + "loss": 1.3357, + "theoretical_loss": 3.454764584413927, + "tokens_seen": 1838481408 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.28757330775260925, + "objective/train/avg_token_score": 0.022113921120762825, + "objective/train/docs_used": 1039769, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9949387907981873, + "objective/train/original_loss": 4.7078962326049805, + "objective/train/theoretical_loss": 3.4547541457497326, + "objective/train/tokens_used": 197406176, + "objective/train/unlikelihood_loss": 0.05575425177812576, + "theoretical_loss": 3.4547541457497326, + "tokens_seen": 1838546944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505050505050505, + "loss": 1.4208, + "theoretical_loss": 3.4547541457497326, + "tokens_seen": 1838546944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504848484848485, + "loss": 1.3145, + "theoretical_loss": 3.4547437075618053, + "tokens_seen": 1838612480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504646464646465, + "loss": 1.4106, + "theoretical_loss": 3.454733269850106, + "tokens_seen": 1838678016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045044444444444445, + "loss": 1.3905, + "theoretical_loss": 3.4547228326145953, + "tokens_seen": 1838743552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045042424242424246, + "loss": 1.6694, + "theoretical_loss": 3.454712395855236, + "tokens_seen": 1838809088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504040404040404, + "loss": 1.5321, + "theoretical_loss": 3.454701959571988, + "tokens_seen": 1838874624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503838383838384, + "loss": 1.6927, + "theoretical_loss": 3.4546915237648137, + "tokens_seen": 1838940160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503636363636364, + "loss": 1.3334, + "theoretical_loss": 3.4546810884336736, + "tokens_seen": 1839005696 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045034343434343434, + "loss": 1.4439, + "theoretical_loss": 3.454670653578529, + "tokens_seen": 1839071232 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045032323232323235, + "loss": 1.5006, + "theoretical_loss": 3.4546602191993427, + "tokens_seen": 1839136768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503030303030303, + "loss": 1.4973, + "theoretical_loss": 3.4546497852960742, + "tokens_seen": 1839202304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502828282828283, + "loss": 1.3218, + "theoretical_loss": 3.4546393518686855, + "tokens_seen": 1839267840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502626262626263, + "loss": 1.3238, + "theoretical_loss": 3.4546289189171375, + "tokens_seen": 1839333376 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045024242424242423, + "loss": 1.2593, + "theoretical_loss": 3.454618486441393, + "tokens_seen": 1839398912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045022222222222225, + "loss": 1.3184, + "theoretical_loss": 3.4546080544414117, + "tokens_seen": 1839464448 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502020202020202, + "loss": 1.264, + "theoretical_loss": 3.4545976229171558, + "tokens_seen": 1839529984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501818181818182, + "loss": 1.0797, + "theoretical_loss": 3.454587191868587, + "tokens_seen": 1839595520 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045016161616161617, + "loss": 1.6226, + "theoretical_loss": 3.454576761295665, + "tokens_seen": 1839661056 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501414141414141, + "loss": 1.6046, + "theoretical_loss": 3.454566331198353, + "tokens_seen": 1839726592 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045012121212121214, + "loss": 1.2764, + "theoretical_loss": 3.4545559015766116, + "tokens_seen": 1839792128 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045010101010101015, + "loss": 1.4158, + "theoretical_loss": 3.4545454724304028, + "tokens_seen": 1839857664 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500808080808081, + "loss": 1.5471, + "theoretical_loss": 3.4545350437596865, + "tokens_seen": 1839923200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045006060606060606, + "loss": 1.4801, + "theoretical_loss": 3.4545246155644254, + "tokens_seen": 1839988736 + }, + { + "epoch": 0.11, + "learning_rate": 0.000450040404040404, + "loss": 1.4026, + "theoretical_loss": 3.4545141878445804, + "tokens_seen": 1840054272 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045002020202020203, + "loss": 1.6834, + "theoretical_loss": 3.4545037606001134, + "tokens_seen": 1840119808 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.6422287225723267, + "objective/train/avg_token_score": 0.03135533630847931, + "objective/train/docs_used": 1040587, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.251701593399048, + "objective/train/original_loss": 3.8327317237854004, + "objective/train/theoretical_loss": 3.4544933338309853, + "objective/train/tokens_used": 199044576, + "objective/train/unlikelihood_loss": 0.05906657129526138, + "theoretical_loss": 3.4544933338309853, + "tokens_seen": 1840185344 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045000000000000004, + "loss": 1.7251, + "theoretical_loss": 3.4544933338309853, + "tokens_seen": 1840185344 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449979797979798, + "loss": 1.4279, + "theoretical_loss": 3.4544829075371575, + "tokens_seen": 1840250880 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044995959595959596, + "loss": 1.5526, + "theoretical_loss": 3.4544724817185912, + "tokens_seen": 1840316416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004499393939393939, + "loss": 1.3022, + "theoretical_loss": 3.454462056375249, + "tokens_seen": 1840381952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004499191919191919, + "loss": 1.6946, + "theoretical_loss": 3.454451631507091, + "tokens_seen": 1840447488 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044989898989898993, + "loss": 1.3439, + "theoretical_loss": 3.454441207114079, + "tokens_seen": 1840513024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498787878787879, + "loss": 1.5178, + "theoretical_loss": 3.454430783196174, + "tokens_seen": 1840578560 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044985858585858585, + "loss": 1.328, + "theoretical_loss": 3.4544203597533385, + "tokens_seen": 1840644096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498383838383838, + "loss": 1.3865, + "theoretical_loss": 3.454409936785533, + "tokens_seen": 1840709632 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044981818181818187, + "loss": 1.4214, + "theoretical_loss": 3.45439951429272, + "tokens_seen": 1840775168 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044979797979797983, + "loss": 1.4532, + "theoretical_loss": 3.4543890922748597, + "tokens_seen": 1840840704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497777777777778, + "loss": 1.7572, + "theoretical_loss": 3.454378670731914, + "tokens_seen": 1840906240 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044975757575757574, + "loss": 1.2789, + "theoretical_loss": 3.454368249663845, + "tokens_seen": 1840971776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497373737373737, + "loss": 1.4031, + "theoretical_loss": 3.454357829070613, + "tokens_seen": 1841037312 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044971717171717176, + "loss": 1.2913, + "theoretical_loss": 3.4543474089521804, + "tokens_seen": 1841102848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496969696969697, + "loss": 1.3861, + "theoretical_loss": 3.4543369893085085, + "tokens_seen": 1841168384 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496767676767677, + "loss": 1.2665, + "theoretical_loss": 3.4543265701395582, + "tokens_seen": 1841233920 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044965656565656563, + "loss": 1.4635, + "theoretical_loss": 3.4543161514452914, + "tokens_seen": 1841299456 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044963636363636364, + "loss": 1.3889, + "theoretical_loss": 3.45430573322567, + "tokens_seen": 1841364992 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044961616161616166, + "loss": 1.5038, + "theoretical_loss": 3.4542953154806546, + "tokens_seen": 1841430528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495959595959596, + "loss": 1.4529, + "theoretical_loss": 3.4542848982102075, + "tokens_seen": 1841496064 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044957575757575757, + "loss": 1.3667, + "theoretical_loss": 3.4542744814142896, + "tokens_seen": 1841561600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495555555555555, + "loss": 1.4548, + "theoretical_loss": 3.454264065092863, + "tokens_seen": 1841627136 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044953535353535354, + "loss": 1.4561, + "theoretical_loss": 3.454253649245888, + "tokens_seen": 1841692672 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044951515151515155, + "loss": 1.3452, + "theoretical_loss": 3.4542432338733278, + "tokens_seen": 1841758208 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.3169599175453186, + "objective/train/avg_token_score": 0.005386400036513805, + "objective/train/docs_used": 1041999, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0743252038955688, + "objective/train/original_loss": 4.4719977378845215, + "objective/train/theoretical_loss": 3.4542328189751426, + "objective/train/tokens_used": 200682976, + "objective/train/unlikelihood_loss": 0.08110522478818893, + "theoretical_loss": 3.4542328189751426, + "tokens_seen": 1841823744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494949494949495, + "loss": 1.1803, + "theoretical_loss": 3.4542328189751426, + "tokens_seen": 1841823744 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044947474747474746, + "loss": 1.474, + "theoretical_loss": 3.4542224045512944, + "tokens_seen": 1841889280 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494545454545455, + "loss": 1.5223, + "theoretical_loss": 3.4542119906017446, + "tokens_seen": 1841954816 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044943434343434343, + "loss": 1.4056, + "theoretical_loss": 3.4542015771264554, + "tokens_seen": 1842020352 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044941414141414144, + "loss": 1.4223, + "theoretical_loss": 3.454191164125387, + "tokens_seen": 1842085888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493939393939394, + "loss": 1.4174, + "theoretical_loss": 3.4541807515985017, + "tokens_seen": 1842151424 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044937373737373735, + "loss": 1.5226, + "theoretical_loss": 3.4541703395457617, + "tokens_seen": 1842216960 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044935353535353537, + "loss": 1.4965, + "theoretical_loss": 3.4541599279671273, + "tokens_seen": 1842282496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493333333333333, + "loss": 1.5692, + "theoretical_loss": 3.454149516862561, + "tokens_seen": 1842348032 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044931313131313133, + "loss": 1.3349, + "theoretical_loss": 3.4541391062320237, + "tokens_seen": 1842413568 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492929292929293, + "loss": 1.4055, + "theoretical_loss": 3.454128696075477, + "tokens_seen": 1842479104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492727272727273, + "loss": 1.3759, + "theoretical_loss": 3.454118286392883, + "tokens_seen": 1842544640 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044925252525252526, + "loss": 1.5596, + "theoretical_loss": 3.454107877184203, + "tokens_seen": 1842610176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492323232323232, + "loss": 1.3979, + "theoretical_loss": 3.4540974684493984, + "tokens_seen": 1842675712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492121212121212, + "loss": 1.6526, + "theoretical_loss": 3.4540870601884306, + "tokens_seen": 1842741248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491919191919192, + "loss": 1.5212, + "theoretical_loss": 3.4540766524012616, + "tokens_seen": 1842806784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491717171717172, + "loss": 1.5532, + "theoretical_loss": 3.454066245087853, + "tokens_seen": 1842872320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044915151515151515, + "loss": 1.5621, + "theoretical_loss": 3.4540558382481663, + "tokens_seen": 1842937856 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044913131313131316, + "loss": 1.3275, + "theoretical_loss": 3.4540454318821627, + "tokens_seen": 1843003392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491111111111111, + "loss": 1.4118, + "theoretical_loss": 3.4540350259898043, + "tokens_seen": 1843068928 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044909090909090913, + "loss": 1.5738, + "theoretical_loss": 3.4540246205710528, + "tokens_seen": 1843134464 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490707070707071, + "loss": 1.5702, + "theoretical_loss": 3.454014215625869, + "tokens_seen": 1843200000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044905050505050504, + "loss": 1.5978, + "theoretical_loss": 3.4540038111542155, + "tokens_seen": 1843265536 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044903030303030306, + "loss": 1.4861, + "theoretical_loss": 3.4539934071560534, + "tokens_seen": 1843331072 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449010101010101, + "loss": 1.5306, + "theoretical_loss": 3.453983003631344, + "tokens_seen": 1843396608 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.30321359634399414, + "objective/train/avg_token_score": 0.03322860598564148, + "objective/train/docs_used": 1042423, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0053961277008057, + "objective/train/original_loss": 4.474796772003174, + "objective/train/theoretical_loss": 3.45397260058005, + "objective/train/tokens_used": 202321376, + "objective/train/unlikelihood_loss": 0.05541876330971718, + "theoretical_loss": 3.45397260058005, + "tokens_seen": 1843462144 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448989898989899, + "loss": 1.5138, + "theoretical_loss": 3.45397260058005, + "tokens_seen": 1843462144 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448969696969697, + "loss": 1.5161, + "theoretical_loss": 3.453962198002132, + "tokens_seen": 1843527680 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044894949494949494, + "loss": 1.2524, + "theoretical_loss": 3.4539517958975514, + "tokens_seen": 1843593216 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044892929292929295, + "loss": 1.5945, + "theoretical_loss": 3.4539413942662707, + "tokens_seen": 1843658752 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044890909090909096, + "loss": 1.5995, + "theoretical_loss": 3.4539309931082514, + "tokens_seen": 1843724288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488888888888889, + "loss": 1.4001, + "theoretical_loss": 3.4539205924234553, + "tokens_seen": 1843789824 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044886868686868687, + "loss": 1.5303, + "theoretical_loss": 3.4539101922118434, + "tokens_seen": 1843855360 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044884848484848483, + "loss": 1.3048, + "theoretical_loss": 3.4538997924733774, + "tokens_seen": 1843920896 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044882828282828284, + "loss": 1.4622, + "theoretical_loss": 3.4538893932080197, + "tokens_seen": 1843986432 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044880808080808085, + "loss": 1.3001, + "theoretical_loss": 3.4538789944157315, + "tokens_seen": 1844051968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487878787878788, + "loss": 1.232, + "theoretical_loss": 3.4538685960964743, + "tokens_seen": 1844117504 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044876767676767677, + "loss": 1.5215, + "theoretical_loss": 3.45385819825021, + "tokens_seen": 1844183040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487474747474747, + "loss": 1.4533, + "theoretical_loss": 3.4538478008769, + "tokens_seen": 1844248576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487272727272728, + "loss": 1.2763, + "theoretical_loss": 3.4538374039765065, + "tokens_seen": 1844314112 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044870707070707074, + "loss": 1.6231, + "theoretical_loss": 3.4538270075489907, + "tokens_seen": 1844379648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486868686868687, + "loss": 1.4151, + "theoretical_loss": 3.4538166115943145, + "tokens_seen": 1844445184 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044866666666666666, + "loss": 1.5396, + "theoretical_loss": 3.45380621611244, + "tokens_seen": 1844510720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486464646464646, + "loss": 1.4981, + "theoretical_loss": 3.453795821103328, + "tokens_seen": 1844576256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486262626262627, + "loss": 1.2735, + "theoretical_loss": 3.4537854265669408, + "tokens_seen": 1844641792 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044860606060606064, + "loss": 1.4933, + "theoretical_loss": 3.4537750325032404, + "tokens_seen": 1844707328 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485858585858586, + "loss": 1.3423, + "theoretical_loss": 3.4537646389121877, + "tokens_seen": 1844772864 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044856565656565655, + "loss": 1.4131, + "theoretical_loss": 3.4537542457937453, + "tokens_seen": 1844838400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485454545454545, + "loss": 1.3256, + "theoretical_loss": 3.4537438531478735, + "tokens_seen": 1844903936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485252525252526, + "loss": 1.3916, + "theoretical_loss": 3.4537334609745356, + "tokens_seen": 1844969472 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044850505050505053, + "loss": 1.5918, + "theoretical_loss": 3.4537230692736927, + "tokens_seen": 1845035008 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.3615591526031494, + "objective/train/avg_token_score": 0.012851543724536896, + "objective/train/docs_used": 1043833, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1701569557189941, + "objective/train/original_loss": 4.571152687072754, + "objective/train/theoretical_loss": 3.4537126780453065, + "objective/train/tokens_used": 203959776, + "objective/train/unlikelihood_loss": 0.05402657762169838, + "theoretical_loss": 3.4537126780453065, + "tokens_seen": 1845100544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484848484848485, + "loss": 1.4921, + "theoretical_loss": 3.4537126780453065, + "tokens_seen": 1845100544 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044846464646464644, + "loss": 1.4343, + "theoretical_loss": 3.453702287289339, + "tokens_seen": 1845166080 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044844444444444445, + "loss": 1.4802, + "theoretical_loss": 3.4536918970057515, + "tokens_seen": 1845231616 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044842424242424247, + "loss": 1.4453, + "theoretical_loss": 3.453681507194506, + "tokens_seen": 1845297152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484040404040404, + "loss": 1.6961, + "theoretical_loss": 3.4536711178555644, + "tokens_seen": 1845362688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483838383838384, + "loss": 1.4932, + "theoretical_loss": 3.453660728988888, + "tokens_seen": 1845428224 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044836363636363634, + "loss": 1.4792, + "theoretical_loss": 3.453650340594439, + "tokens_seen": 1845493760 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044834343434343435, + "loss": 1.454, + "theoretical_loss": 3.453639952672179, + "tokens_seen": 1845559296 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044832323232323236, + "loss": 1.5949, + "theoretical_loss": 3.4536295652220694, + "tokens_seen": 1845624832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483030303030303, + "loss": 1.4518, + "theoretical_loss": 3.453619178244073, + "tokens_seen": 1845690368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044828282828282827, + "loss": 1.5919, + "theoretical_loss": 3.4536087917381506, + "tokens_seen": 1845755904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482626262626263, + "loss": 1.6802, + "theoretical_loss": 3.4535984057042644, + "tokens_seen": 1845821440 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044824242424242424, + "loss": 1.3382, + "theoretical_loss": 3.4535880201423765, + "tokens_seen": 1845886976 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044822222222222225, + "loss": 1.5491, + "theoretical_loss": 3.4535776350524476, + "tokens_seen": 1845952512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482020202020202, + "loss": 1.3793, + "theoretical_loss": 3.453567250434441, + "tokens_seen": 1846018048 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044818181818181816, + "loss": 1.5925, + "theoretical_loss": 3.453556866288317, + "tokens_seen": 1846083584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481616161616162, + "loss": 1.5228, + "theoretical_loss": 3.453546482614038, + "tokens_seen": 1846149120 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044814141414141413, + "loss": 1.5559, + "theoretical_loss": 3.4535360994115667, + "tokens_seen": 1846214656 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044812121212121214, + "loss": 1.5943, + "theoretical_loss": 3.453525716680863, + "tokens_seen": 1846280192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481010101010101, + "loss": 1.4601, + "theoretical_loss": 3.453515334421891, + "tokens_seen": 1846345728 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004480808080808081, + "loss": 1.3743, + "theoretical_loss": 3.453504952634611, + "tokens_seen": 1846411264 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044806060606060607, + "loss": 1.4953, + "theoretical_loss": 3.453494571318985, + "tokens_seen": 1846476800 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448040404040404, + "loss": 1.5075, + "theoretical_loss": 3.453484190474976, + "tokens_seen": 1846542336 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044802020202020204, + "loss": 1.4881, + "theoretical_loss": 3.4534738101025435, + "tokens_seen": 1846607872 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448, + "loss": 1.2744, + "theoretical_loss": 3.4534634302016514, + "tokens_seen": 1846673408 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.4943181872367859, + "objective/train/avg_token_score": 0.03060787171125412, + "objective/train/docs_used": 1044447, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6434940099716187, + "objective/train/original_loss": 4.084818363189697, + "objective/train/theoretical_loss": 3.4534530507722607, + "objective/train/tokens_used": 205598176, + "objective/train/unlikelihood_loss": 0.05918062478303909, + "theoretical_loss": 3.4534530507722607, + "tokens_seen": 1846738944 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447979797979798, + "loss": 1.5188, + "theoretical_loss": 3.4534530507722607, + "tokens_seen": 1846738944 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044795959595959596, + "loss": 1.5925, + "theoretical_loss": 3.453442671814334, + "tokens_seen": 1846804480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004479393939393939, + "loss": 1.3562, + "theoretical_loss": 3.4534322933278316, + "tokens_seen": 1846870016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044791919191919193, + "loss": 1.4972, + "theoretical_loss": 3.453421915312717, + "tokens_seen": 1846935552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044789898989898994, + "loss": 1.306, + "theoretical_loss": 3.453411537768951, + "tokens_seen": 1847001088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478787878787879, + "loss": 1.3592, + "theoretical_loss": 3.4534011606964965, + "tokens_seen": 1847066624 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044785858585858585, + "loss": 1.6231, + "theoretical_loss": 3.4533907840953146, + "tokens_seen": 1847132160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478383838383838, + "loss": 1.5106, + "theoretical_loss": 3.453380407965367, + "tokens_seen": 1847197696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478181818181818, + "loss": 1.438, + "theoretical_loss": 3.4533700323066157, + "tokens_seen": 1847263232 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044779797979797983, + "loss": 1.3438, + "theoretical_loss": 3.4533596571190235, + "tokens_seen": 1847328768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477777777777778, + "loss": 1.3831, + "theoretical_loss": 3.4533492824025513, + "tokens_seen": 1847394304 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044775757575757575, + "loss": 1.423, + "theoretical_loss": 3.453338908157161, + "tokens_seen": 1847459840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477373737373737, + "loss": 1.5044, + "theoretical_loss": 3.4533285343828153, + "tokens_seen": 1847525376 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044771717171717177, + "loss": 1.5387, + "theoretical_loss": 3.453318161079475, + "tokens_seen": 1847590912 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476969696969697, + "loss": 1.2799, + "theoretical_loss": 3.453307788247103, + "tokens_seen": 1847656448 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476767676767677, + "loss": 1.5208, + "theoretical_loss": 3.45329741588566, + "tokens_seen": 1847721984 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044765656565656564, + "loss": 1.6915, + "theoretical_loss": 3.4532870439951098, + "tokens_seen": 1847787520 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476363636363636, + "loss": 1.7828, + "theoretical_loss": 3.453276672575413, + "tokens_seen": 1847853056 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044761616161616166, + "loss": 1.515, + "theoretical_loss": 3.4532663016265315, + "tokens_seen": 1847918592 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475959595959596, + "loss": 1.3178, + "theoretical_loss": 3.4532559311484277, + "tokens_seen": 1847984128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475757575757576, + "loss": 1.4015, + "theoretical_loss": 3.4532455611410633, + "tokens_seen": 1848049664 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044755555555555553, + "loss": 1.3752, + "theoretical_loss": 3.4532351916044, + "tokens_seen": 1848115200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044753535353535354, + "loss": 1.4995, + "theoretical_loss": 3.4532248225384006, + "tokens_seen": 1848180736 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044751515151515155, + "loss": 1.2319, + "theoretical_loss": 3.453214453943026, + "tokens_seen": 1848246272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474949494949495, + "loss": 1.3379, + "theoretical_loss": 3.4532040858182387, + "tokens_seen": 1848311808 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.3084677457809448, + "objective/train/avg_token_score": 0.008858302608132362, + "objective/train/docs_used": 1045763, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8898696899414062, + "objective/train/original_loss": 4.509169101715088, + "objective/train/theoretical_loss": 3.453193718164001, + "objective/train/tokens_used": 207236576, + "objective/train/unlikelihood_loss": 0.06134689226746559, + "theoretical_loss": 3.453193718164001, + "tokens_seen": 1848377344 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044747474747474747, + "loss": 1.286, + "theoretical_loss": 3.453193718164001, + "tokens_seen": 1848377344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474545454545454, + "loss": 1.4064, + "theoretical_loss": 3.4531833509802734, + "tokens_seen": 1848442880 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044743434343434344, + "loss": 1.3498, + "theoretical_loss": 3.45317298426702, + "tokens_seen": 1848508416 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044741414141414145, + "loss": 1.1875, + "theoretical_loss": 3.453162618024201, + "tokens_seen": 1848573952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473939393939394, + "loss": 1.5256, + "theoretical_loss": 3.4531522522517797, + "tokens_seen": 1848639488 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044737373737373736, + "loss": 1.5795, + "theoretical_loss": 3.453141886949717, + "tokens_seen": 1848705024 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044735353535353537, + "loss": 1.4829, + "theoretical_loss": 3.453131522117975, + "tokens_seen": 1848770560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473333333333334, + "loss": 1.6749, + "theoretical_loss": 3.4531211577565166, + "tokens_seen": 1848836096 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044731313131313134, + "loss": 1.5497, + "theoretical_loss": 3.453110793865303, + "tokens_seen": 1848901632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472929292929293, + "loss": 1.431, + "theoretical_loss": 3.4531004304442963, + "tokens_seen": 1848967168 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044727272727272725, + "loss": 1.686, + "theoretical_loss": 3.4530900674934584, + "tokens_seen": 1849032704 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044725252525252526, + "loss": 1.3995, + "theoretical_loss": 3.453079705012752, + "tokens_seen": 1849098240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472323232323233, + "loss": 1.5956, + "theoretical_loss": 3.4530693430021384, + "tokens_seen": 1849163776 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044721212121212123, + "loss": 1.6193, + "theoretical_loss": 3.45305898146158, + "tokens_seen": 1849229312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471919191919192, + "loss": 1.3201, + "theoretical_loss": 3.453048620391038, + "tokens_seen": 1849294848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471717171717172, + "loss": 1.2248, + "theoretical_loss": 3.4530382597904756, + "tokens_seen": 1849360384 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044715151515151516, + "loss": 1.2997, + "theoretical_loss": 3.4530278996598547, + "tokens_seen": 1849425920 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044713131313131317, + "loss": 1.3588, + "theoretical_loss": 3.453017539999136, + "tokens_seen": 1849491456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471111111111111, + "loss": 1.5867, + "theoretical_loss": 3.4530071808082834, + "tokens_seen": 1849556992 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470909090909091, + "loss": 1.5271, + "theoretical_loss": 3.4529968220872576, + "tokens_seen": 1849622528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470707070707071, + "loss": 1.3612, + "theoretical_loss": 3.4529864638360213, + "tokens_seen": 1849688064 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044705050505050505, + "loss": 1.7033, + "theoretical_loss": 3.452976106054536, + "tokens_seen": 1849753600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044703030303030306, + "loss": 1.5997, + "theoretical_loss": 3.4529657487427636, + "tokens_seen": 1849819136 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447010101010101, + "loss": 1.2692, + "theoretical_loss": 3.4529553919006672, + "tokens_seen": 1849884672 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044698989898989903, + "loss": 1.4866, + "theoretical_loss": 3.4529450355282085, + "tokens_seen": 1849950208 + }, + { + "epoch": 0.11, + "objective/train/aligned_token_count": 0.5390396118164062, + "objective/train/avg_token_score": 0.004928861279040575, + "objective/train/docs_used": 1046397, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7157353162765503, + "objective/train/original_loss": 3.4299583435058594, + "objective/train/theoretical_loss": 3.452934679625349, + "objective/train/tokens_used": 208874976, + "objective/train/unlikelihood_loss": 0.08482911437749863, + "theoretical_loss": 3.452934679625349, + "tokens_seen": 1850015744 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446969696969697, + "loss": 1.5823, + "theoretical_loss": 3.452934679625349, + "tokens_seen": 1850015744 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044694949494949494, + "loss": 1.4851, + "theoretical_loss": 3.4529243241920513, + "tokens_seen": 1850081280 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044692929292929295, + "loss": 1.5261, + "theoretical_loss": 3.452913969228277, + "tokens_seen": 1850146816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469090909090909, + "loss": 1.4711, + "theoretical_loss": 3.4529036147339887, + "tokens_seen": 1850212352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468888888888889, + "loss": 1.5649, + "theoretical_loss": 3.452893260709148, + "tokens_seen": 1850277888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468686868686869, + "loss": 1.5514, + "theoretical_loss": 3.4528829071537177, + "tokens_seen": 1850343424 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044684848484848484, + "loss": 1.3146, + "theoretical_loss": 3.4528725540676595, + "tokens_seen": 1850408960 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044682828282828285, + "loss": 1.5784, + "theoretical_loss": 3.452862201450935, + "tokens_seen": 1850474496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468080808080808, + "loss": 1.7336, + "theoretical_loss": 3.452851849303507, + "tokens_seen": 1850540032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467878787878788, + "loss": 1.4407, + "theoretical_loss": 3.452841497625337, + "tokens_seen": 1850605568 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044676767676767677, + "loss": 1.1921, + "theoretical_loss": 3.452831146416388, + "tokens_seen": 1850671104 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044674747474747473, + "loss": 1.4424, + "theoretical_loss": 3.4528207956766215, + "tokens_seen": 1850736640 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044672727272727274, + "loss": 1.6409, + "theoretical_loss": 3.4528104454059996, + "tokens_seen": 1850802176 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044670707070707075, + "loss": 1.4714, + "theoretical_loss": 3.4528000956044846, + "tokens_seen": 1850867712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466868686868687, + "loss": 1.3572, + "theoretical_loss": 3.4527897462720385, + "tokens_seen": 1850933248 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044666666666666666, + "loss": 1.2197, + "theoretical_loss": 3.452779397408624, + "tokens_seen": 1850998784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466464646464646, + "loss": 1.4201, + "theoretical_loss": 3.452769049014202, + "tokens_seen": 1851064320 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044662626262626263, + "loss": 1.4719, + "theoretical_loss": 3.4527587010887357, + "tokens_seen": 1851129856 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044660606060606064, + "loss": 1.4856, + "theoretical_loss": 3.4527483536321864, + "tokens_seen": 1851195392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465858585858586, + "loss": 1.5586, + "theoretical_loss": 3.4527380066445175, + "tokens_seen": 1851260928 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044656565656565656, + "loss": 1.4921, + "theoretical_loss": 3.4527276601256904, + "tokens_seen": 1851326464 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465454545454545, + "loss": 1.3876, + "theoretical_loss": 3.4527173140756666, + "tokens_seen": 1851392000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465252525252526, + "loss": 1.6662, + "theoretical_loss": 3.4527069684944096, + "tokens_seen": 1851457536 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044650505050505054, + "loss": 1.3775, + "theoretical_loss": 3.4526966233818808, + "tokens_seen": 1851523072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464848484848485, + "loss": 1.3225, + "theoretical_loss": 3.4526862787380423, + "tokens_seen": 1851588608 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.2721163332462311, + "objective/train/avg_token_score": 0.026380645111203194, + "objective/train/docs_used": 1047383, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.913923442363739, + "objective/train/original_loss": 4.239612102508545, + "objective/train/theoretical_loss": 3.4526759345628566, + "objective/train/tokens_used": 210513376, + "objective/train/unlikelihood_loss": 0.079360231757164, + "theoretical_loss": 3.4526759345628566, + "tokens_seen": 1851654144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044646464646464645, + "loss": 1.4303, + "theoretical_loss": 3.4526759345628566, + "tokens_seen": 1851654144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464444444444444, + "loss": 1.2667, + "theoretical_loss": 3.452665590856286, + "tokens_seen": 1851719680 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044642424242424247, + "loss": 1.4178, + "theoretical_loss": 3.452655247618292, + "tokens_seen": 1851785216 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044640404040404043, + "loss": 1.3794, + "theoretical_loss": 3.452644904848838, + "tokens_seen": 1851850752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463838383838384, + "loss": 1.5685, + "theoretical_loss": 3.4526345625478845, + "tokens_seen": 1851916288 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044636363636363634, + "loss": 1.553, + "theoretical_loss": 3.4526242207153954, + "tokens_seen": 1851981824 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044634343434343435, + "loss": 1.7237, + "theoretical_loss": 3.4526138793513317, + "tokens_seen": 1852047360 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044632323232323236, + "loss": 1.753, + "theoretical_loss": 3.452603538455656, + "tokens_seen": 1852112896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463030303030303, + "loss": 1.4571, + "theoretical_loss": 3.452593198028331, + "tokens_seen": 1852178432 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462828282828283, + "loss": 1.31, + "theoretical_loss": 3.4525828580693183, + "tokens_seen": 1852243968 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044626262626262623, + "loss": 1.5186, + "theoretical_loss": 3.45257251857858, + "tokens_seen": 1852309504 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044624242424242425, + "loss": 1.6351, + "theoretical_loss": 3.452562179556079, + "tokens_seen": 1852375040 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044622222222222226, + "loss": 1.1539, + "theoretical_loss": 3.452551841001777, + "tokens_seen": 1852440576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462020202020202, + "loss": 1.3132, + "theoretical_loss": 3.4525415029156363, + "tokens_seen": 1852506112 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044618181818181817, + "loss": 1.2812, + "theoretical_loss": 3.452531165297619, + "tokens_seen": 1852571648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461616161616162, + "loss": 1.606, + "theoretical_loss": 3.452520828147688, + "tokens_seen": 1852637184 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044614141414141414, + "loss": 1.4788, + "theoretical_loss": 3.452510491465805, + "tokens_seen": 1852702720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044612121212121215, + "loss": 1.5314, + "theoretical_loss": 3.4525001552519323, + "tokens_seen": 1852768256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461010101010101, + "loss": 1.309, + "theoretical_loss": 3.4524898195060323, + "tokens_seen": 1852833792 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044608080808080806, + "loss": 1.384, + "theoretical_loss": 3.4524794842280673, + "tokens_seen": 1852899328 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460606060606061, + "loss": 1.462, + "theoretical_loss": 3.4524691494179995, + "tokens_seen": 1852964864 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044604040404040403, + "loss": 1.6896, + "theoretical_loss": 3.452458815075791, + "tokens_seen": 1853030400 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044602020202020204, + "loss": 1.4372, + "theoretical_loss": 3.452448481201404, + "tokens_seen": 1853095936 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446, + "loss": 1.4209, + "theoretical_loss": 3.452438147794801, + "tokens_seen": 1853161472 + }, + { + "epoch": 0.12, + "learning_rate": 0.000445979797979798, + "loss": 1.422, + "theoretical_loss": 3.4524278148559446, + "tokens_seen": 1853227008 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.31262218952178955, + "objective/train/avg_token_score": 0.019027452915906906, + "objective/train/docs_used": 1048172, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1232824325561523, + "objective/train/original_loss": 4.744579792022705, + "objective/train/theoretical_loss": 3.4524174823847966, + "objective/train/tokens_used": 212151776, + "objective/train/unlikelihood_loss": 0.04645653814077377, + "theoretical_loss": 3.4524174823847966, + "tokens_seen": 1853292544 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044595959595959597, + "loss": 1.1746, + "theoretical_loss": 3.4524174823847966, + "tokens_seen": 1853292544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459393939393939, + "loss": 1.4012, + "theoretical_loss": 3.452407150381319, + "tokens_seen": 1853358080 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044591919191919193, + "loss": 1.4507, + "theoretical_loss": 3.4523968188454752, + "tokens_seen": 1853423616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458989898989899, + "loss": 1.463, + "theoretical_loss": 3.452386487777227, + "tokens_seen": 1853489152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458787878787879, + "loss": 1.4316, + "theoretical_loss": 3.452376157176536, + "tokens_seen": 1853554688 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044585858585858586, + "loss": 1.7792, + "theoretical_loss": 3.452365827043365, + "tokens_seen": 1853620224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458383838383838, + "loss": 1.6054, + "theoretical_loss": 3.4523554973776767, + "tokens_seen": 1853685760 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044581818181818183, + "loss": 1.3471, + "theoretical_loss": 3.452345168179433, + "tokens_seen": 1853751296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044579797979797984, + "loss": 1.4412, + "theoretical_loss": 3.4523348394485964, + "tokens_seen": 1853816832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457777777777778, + "loss": 1.3501, + "theoretical_loss": 3.452324511185129, + "tokens_seen": 1853882368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044575757575757575, + "loss": 1.5304, + "theoretical_loss": 3.452314183388993, + "tokens_seen": 1853947904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457373737373737, + "loss": 1.4022, + "theoretical_loss": 3.4523038560601513, + "tokens_seen": 1854013440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457171717171717, + "loss": 1.3444, + "theoretical_loss": 3.452293529198566, + "tokens_seen": 1854078976 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044569696969696973, + "loss": 1.2717, + "theoretical_loss": 3.4522832028041988, + "tokens_seen": 1854144512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456767676767677, + "loss": 1.3764, + "theoretical_loss": 3.4522728768770135, + "tokens_seen": 1854210048 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044565656565656564, + "loss": 1.3803, + "theoretical_loss": 3.452262551416971, + "tokens_seen": 1854275584 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044563636363636366, + "loss": 1.4479, + "theoretical_loss": 3.4522522264240347, + "tokens_seen": 1854341120 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044561616161616167, + "loss": 1.6955, + "theoretical_loss": 3.452241901898166, + "tokens_seen": 1854406656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455959595959596, + "loss": 1.6218, + "theoretical_loss": 3.452231577839328, + "tokens_seen": 1854472192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455757575757576, + "loss": 1.3389, + "theoretical_loss": 3.452221254247483, + "tokens_seen": 1854537728 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044555555555555554, + "loss": 1.521, + "theoretical_loss": 3.4522109311225933, + "tokens_seen": 1854603264 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044553535353535355, + "loss": 1.2394, + "theoretical_loss": 3.4522006084646204, + "tokens_seen": 1854668800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044551515151515156, + "loss": 1.4995, + "theoretical_loss": 3.4521902862735283, + "tokens_seen": 1854734336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454949494949495, + "loss": 1.3931, + "theoretical_loss": 3.4521799645492783, + "tokens_seen": 1854799872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454747474747475, + "loss": 1.5762, + "theoretical_loss": 3.452169643291833, + "tokens_seen": 1854865408 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.40560850501060486, + "objective/train/avg_token_score": 0.008102362044155598, + "objective/train/docs_used": 1049323, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3408201932907104, + "objective/train/original_loss": 4.186563968658447, + "objective/train/theoretical_loss": 3.452159322501155, + "objective/train/tokens_used": 213790176, + "objective/train/unlikelihood_loss": 0.057345665991306305, + "theoretical_loss": 3.452159322501155, + "tokens_seen": 1854930944 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044545454545454543, + "loss": 1.5162, + "theoretical_loss": 3.452159322501155, + "tokens_seen": 1854930944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454343434343435, + "loss": 1.5745, + "theoretical_loss": 3.4521490021772063, + "tokens_seen": 1854996480 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044541414141414145, + "loss": 1.3886, + "theoretical_loss": 3.45213868231995, + "tokens_seen": 1855062016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453939393939394, + "loss": 1.381, + "theoretical_loss": 3.452128362929348, + "tokens_seen": 1855127552 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044537373737373737, + "loss": 1.703, + "theoretical_loss": 3.4521180440053625, + "tokens_seen": 1855193088 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453535353535353, + "loss": 1.569, + "theoretical_loss": 3.452107725547956, + "tokens_seen": 1855258624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453333333333334, + "loss": 1.2288, + "theoretical_loss": 3.4520974075570914, + "tokens_seen": 1855324160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044531313131313135, + "loss": 1.5825, + "theoretical_loss": 3.452087090032731, + "tokens_seen": 1855389696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004452929292929293, + "loss": 1.5339, + "theoretical_loss": 3.452076772974837, + "tokens_seen": 1855455232 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044527272727272726, + "loss": 1.173, + "theoretical_loss": 3.4520664563833723, + "tokens_seen": 1855520768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004452525252525252, + "loss": 1.5338, + "theoretical_loss": 3.452056140258298, + "tokens_seen": 1855586304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004452323232323233, + "loss": 1.2947, + "theoretical_loss": 3.4520458245995784, + "tokens_seen": 1855651840 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044521212121212124, + "loss": 1.4635, + "theoretical_loss": 3.4520355094071746, + "tokens_seen": 1855717376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451919191919192, + "loss": 1.2195, + "theoretical_loss": 3.45202519468105, + "tokens_seen": 1855782912 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044517171717171715, + "loss": 1.5204, + "theoretical_loss": 3.452014880421166, + "tokens_seen": 1855848448 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044515151515151516, + "loss": 1.2559, + "theoretical_loss": 3.452004566627486, + "tokens_seen": 1855913984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451313131313132, + "loss": 1.3033, + "theoretical_loss": 3.4519942532999712, + "tokens_seen": 1855979520 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044511111111111113, + "loss": 1.5715, + "theoretical_loss": 3.451983940438586, + "tokens_seen": 1856045056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450909090909091, + "loss": 1.506, + "theoretical_loss": 3.4519736280432918, + "tokens_seen": 1856110592 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044507070707070704, + "loss": 1.4881, + "theoretical_loss": 3.4519633161140506, + "tokens_seen": 1856176128 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044505050505050506, + "loss": 1.3911, + "theoretical_loss": 3.4519530046508256, + "tokens_seen": 1856241664 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044503030303030307, + "loss": 1.3362, + "theoretical_loss": 3.451942693653579, + "tokens_seen": 1856307200 + }, + { + "epoch": 0.12, + "learning_rate": 0.000445010101010101, + "loss": 1.4433, + "theoretical_loss": 3.4519323831222737, + "tokens_seen": 1856372736 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444989898989899, + "loss": 1.2808, + "theoretical_loss": 3.4519220730568714, + "tokens_seen": 1856438272 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444969696969697, + "loss": 1.43, + "theoretical_loss": 3.4519117634573355, + "tokens_seen": 1856503808 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.33345553278923035, + "objective/train/avg_token_score": 0.06058519333600998, + "objective/train/docs_used": 1049929, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.12827730178833, + "objective/train/original_loss": 4.973564624786377, + "objective/train/theoretical_loss": 3.451901454323628, + "objective/train/tokens_used": 215428576, + "objective/train/unlikelihood_loss": 0.046354275196790695, + "theoretical_loss": 3.451901454323628, + "tokens_seen": 1856569344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044494949494949495, + "loss": 1.4263, + "theoretical_loss": 3.451901454323628, + "tokens_seen": 1856569344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044492929292929296, + "loss": 1.2963, + "theoretical_loss": 3.4518911456557113, + "tokens_seen": 1856634880 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449090909090909, + "loss": 1.6288, + "theoretical_loss": 3.4518808374535483, + "tokens_seen": 1856700416 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044488888888888887, + "loss": 1.3875, + "theoretical_loss": 3.4518705297171013, + "tokens_seen": 1856765952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448686868686869, + "loss": 1.3577, + "theoretical_loss": 3.4518602224463324, + "tokens_seen": 1856831488 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044484848484848484, + "loss": 1.5654, + "theoretical_loss": 3.4518499156412052, + "tokens_seen": 1856897024 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044482828282828285, + "loss": 1.2628, + "theoretical_loss": 3.4518396093016817, + "tokens_seen": 1856962560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448080808080808, + "loss": 1.3431, + "theoretical_loss": 3.451829303427724, + "tokens_seen": 1857028096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004447878787878788, + "loss": 1.4391, + "theoretical_loss": 3.451818998019295, + "tokens_seen": 1857093632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004447676767676768, + "loss": 1.4135, + "theoretical_loss": 3.451808693076357, + "tokens_seen": 1857159168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044474747474747473, + "loss": 1.5886, + "theoretical_loss": 3.4517983885988732, + "tokens_seen": 1857224704 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044472727272727274, + "loss": 1.4285, + "theoretical_loss": 3.4517880845868056, + "tokens_seen": 1857290240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004447070707070707, + "loss": 1.4832, + "theoretical_loss": 3.4517777810401165, + "tokens_seen": 1857355776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446868686868687, + "loss": 1.3972, + "theoretical_loss": 3.4517674779587693, + "tokens_seen": 1857421312 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044466666666666667, + "loss": 1.4377, + "theoretical_loss": 3.451757175342726, + "tokens_seen": 1857486848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446464646464646, + "loss": 1.5561, + "theoretical_loss": 3.451746873191949, + "tokens_seen": 1857552384 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044462626262626264, + "loss": 1.3046, + "theoretical_loss": 3.4517365715064017, + "tokens_seen": 1857617920 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044460606060606065, + "loss": 1.5743, + "theoretical_loss": 3.451726270286046, + "tokens_seen": 1857683456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445858585858586, + "loss": 1.238, + "theoretical_loss": 3.451715969530844, + "tokens_seen": 1857748992 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044456565656565656, + "loss": 1.6605, + "theoretical_loss": 3.4517056692407597, + "tokens_seen": 1857814528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445454545454545, + "loss": 1.5474, + "theoretical_loss": 3.451695369415755, + "tokens_seen": 1857880064 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044452525252525253, + "loss": 1.4325, + "theoretical_loss": 3.4516850700557917, + "tokens_seen": 1857945600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044450505050505054, + "loss": 1.4822, + "theoretical_loss": 3.451674771160833, + "tokens_seen": 1858011136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444848484848485, + "loss": 1.3059, + "theoretical_loss": 3.451664472730842, + "tokens_seen": 1858076672 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044446464646464645, + "loss": 1.3968, + "theoretical_loss": 3.451654174765781, + "tokens_seen": 1858142208 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.40243157744407654, + "objective/train/avg_token_score": 0.01305119413882494, + "objective/train/docs_used": 1050597, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3890475034713745, + "objective/train/original_loss": 4.159512042999268, + "objective/train/theoretical_loss": 3.451643877265613, + "objective/train/tokens_used": 217066976, + "objective/train/unlikelihood_loss": 0.06591649353504181, + "theoretical_loss": 3.451643877265613, + "tokens_seen": 1858207744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444444444444444, + "loss": 1.5441, + "theoretical_loss": 3.451643877265613, + "tokens_seen": 1858207744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444242424242425, + "loss": 1.4623, + "theoretical_loss": 3.451633580230299, + "tokens_seen": 1858273280 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044440404040404043, + "loss": 1.5315, + "theoretical_loss": 3.4516232836598033, + "tokens_seen": 1858338816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443838383838384, + "loss": 1.2582, + "theoretical_loss": 3.451612987554088, + "tokens_seen": 1858404352 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044436363636363635, + "loss": 1.5215, + "theoretical_loss": 3.451602691913116, + "tokens_seen": 1858469888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443434343434343, + "loss": 1.3035, + "theoretical_loss": 3.4515923967368494, + "tokens_seen": 1858535424 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044432323232323237, + "loss": 1.342, + "theoretical_loss": 3.4515821020252506, + "tokens_seen": 1858600960 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443030303030303, + "loss": 1.3374, + "theoretical_loss": 3.4515718077782838, + "tokens_seen": 1858666496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004442828282828283, + "loss": 1.3353, + "theoretical_loss": 3.4515615139959097, + "tokens_seen": 1858732032 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044426262626262624, + "loss": 1.4248, + "theoretical_loss": 3.451551220678092, + "tokens_seen": 1858797568 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044424242424242425, + "loss": 1.47, + "theoretical_loss": 3.4515409278247935, + "tokens_seen": 1858863104 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044422222222222226, + "loss": 1.1401, + "theoretical_loss": 3.4515306354359767, + "tokens_seen": 1858928640 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004442020202020202, + "loss": 1.5552, + "theoretical_loss": 3.4515203435116035, + "tokens_seen": 1858994176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441818181818182, + "loss": 1.4739, + "theoretical_loss": 3.4515100520516375, + "tokens_seen": 1859059712 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044416161616161613, + "loss": 1.4808, + "theoretical_loss": 3.4514997610560414, + "tokens_seen": 1859125248 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044414141414141414, + "loss": 1.3743, + "theoretical_loss": 3.451489470524777, + "tokens_seen": 1859190784 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044412121212121215, + "loss": 1.4206, + "theoretical_loss": 3.4514791804578078, + "tokens_seen": 1859256320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441010101010101, + "loss": 1.5322, + "theoretical_loss": 3.4514688908550966, + "tokens_seen": 1859321856 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044408080808080807, + "loss": 1.4228, + "theoretical_loss": 3.451458601716605, + "tokens_seen": 1859387392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440606060606061, + "loss": 1.7084, + "theoretical_loss": 3.451448313042297, + "tokens_seen": 1859452928 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044404040404040404, + "loss": 1.1568, + "theoretical_loss": 3.451438024832134, + "tokens_seen": 1859518464 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044402020202020205, + "loss": 1.3594, + "theoretical_loss": 3.45142773708608, + "tokens_seen": 1859584000 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444, + "loss": 1.5296, + "theoretical_loss": 3.451417449804097, + "tokens_seen": 1859649536 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044397979797979796, + "loss": 1.243, + "theoretical_loss": 3.451407162986148, + "tokens_seen": 1859715072 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044395959595959597, + "loss": 1.3292, + "theoretical_loss": 3.451396876632195, + "tokens_seen": 1859780608 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.3087732195854187, + "objective/train/avg_token_score": 0.015457415953278542, + "objective/train/docs_used": 1051522, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0185426473617554, + "objective/train/original_loss": 4.571277618408203, + "objective/train/theoretical_loss": 3.4513865907422017, + "objective/train/tokens_used": 218705376, + "objective/train/unlikelihood_loss": 0.05881958827376366, + "theoretical_loss": 3.4513865907422017, + "tokens_seen": 1859846144 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443939393939394, + "loss": 1.3964, + "theoretical_loss": 3.4513865907422017, + "tokens_seen": 1859846144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044391919191919194, + "loss": 1.4011, + "theoretical_loss": 3.4513763053161304, + "tokens_seen": 1859911680 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438989898989899, + "loss": 1.6454, + "theoretical_loss": 3.4513660203539436, + "tokens_seen": 1859977216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438787878787879, + "loss": 1.4499, + "theoretical_loss": 3.4513557358556044, + "tokens_seen": 1860042752 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044385858585858587, + "loss": 1.4571, + "theoretical_loss": 3.4513454518210755, + "tokens_seen": 1860108288 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438383838383839, + "loss": 1.1652, + "theoretical_loss": 3.4513351682503193, + "tokens_seen": 1860173824 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044381818181818183, + "loss": 1.4164, + "theoretical_loss": 3.4513248851432987, + "tokens_seen": 1860239360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437979797979798, + "loss": 1.2335, + "theoretical_loss": 3.451314602499977, + "tokens_seen": 1860304896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437777777777778, + "loss": 1.4228, + "theoretical_loss": 3.451304320320316, + "tokens_seen": 1860370432 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044375757575757576, + "loss": 1.3746, + "theoretical_loss": 3.451294038604279, + "tokens_seen": 1860435968 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044373737373737377, + "loss": 1.6736, + "theoretical_loss": 3.4512837573518294, + "tokens_seen": 1860501504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437171717171717, + "loss": 1.1772, + "theoretical_loss": 3.4512734765629283, + "tokens_seen": 1860567040 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044369696969696974, + "loss": 1.406, + "theoretical_loss": 3.45126319623754, + "tokens_seen": 1860632576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436767676767677, + "loss": 1.4388, + "theoretical_loss": 3.451252916375627, + "tokens_seen": 1860698112 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044365656565656565, + "loss": 1.6811, + "theoretical_loss": 3.4512426369771516, + "tokens_seen": 1860763648 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044363636363636366, + "loss": 1.3693, + "theoretical_loss": 3.4512323580420765, + "tokens_seen": 1860829184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436161616161616, + "loss": 1.4087, + "theoretical_loss": 3.451222079570365, + "tokens_seen": 1860894720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044359595959595963, + "loss": 1.3867, + "theoretical_loss": 3.4512118015619797, + "tokens_seen": 1860960256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435757575757576, + "loss": 1.566, + "theoretical_loss": 3.4512015240168834, + "tokens_seen": 1861025792 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044355555555555554, + "loss": 1.513, + "theoretical_loss": 3.451191246935039, + "tokens_seen": 1861091328 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044353535353535355, + "loss": 1.492, + "theoretical_loss": 3.4511809703164094, + "tokens_seen": 1861156864 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435151515151515, + "loss": 1.5802, + "theoretical_loss": 3.4511706941609566, + "tokens_seen": 1861222400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434949494949495, + "loss": 1.3848, + "theoretical_loss": 3.4511604184686444, + "tokens_seen": 1861287936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434747474747475, + "loss": 1.4916, + "theoretical_loss": 3.4511501432394356, + "tokens_seen": 1861353472 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044345454545454544, + "loss": 1.1949, + "theoretical_loss": 3.451139868473292, + "tokens_seen": 1861419008 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.37634408473968506, + "objective/train/avg_token_score": 0.01542601641267538, + "objective/train/docs_used": 1052031, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.181724190711975, + "objective/train/original_loss": 4.738986968994141, + "objective/train/theoretical_loss": 3.4511295941701774, + "objective/train/tokens_used": 220343776, + "objective/train/unlikelihood_loss": 0.061755482107400894, + "theoretical_loss": 3.4511295941701774, + "tokens_seen": 1861484544 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044343434343434345, + "loss": 1.2778, + "theoretical_loss": 3.4511295941701774, + "tokens_seen": 1861484544 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044341414141414146, + "loss": 1.4889, + "theoretical_loss": 3.4511193203300543, + "tokens_seen": 1861550080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004433939393939394, + "loss": 1.6773, + "theoretical_loss": 3.451109046952886, + "tokens_seen": 1861615616 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044337373737373737, + "loss": 1.4959, + "theoretical_loss": 3.451098774038634, + "tokens_seen": 1861681152 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044335353535353533, + "loss": 1.5357, + "theoretical_loss": 3.4510885015872628, + "tokens_seen": 1861746688 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044333333333333334, + "loss": 1.1331, + "theoretical_loss": 3.4510782295987346, + "tokens_seen": 1861812224 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044331313131313135, + "loss": 1.1775, + "theoretical_loss": 3.4510679580730117, + "tokens_seen": 1861877760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432929292929293, + "loss": 1.4438, + "theoretical_loss": 3.4510576870100573, + "tokens_seen": 1861943296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044327272727272726, + "loss": 1.2656, + "theoretical_loss": 3.4510474164098346, + "tokens_seen": 1862008832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432525252525252, + "loss": 1.3183, + "theoretical_loss": 3.4510371462723066, + "tokens_seen": 1862074368 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432323232323233, + "loss": 1.4267, + "theoretical_loss": 3.4510268765974357, + "tokens_seen": 1862139904 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044321212121212124, + "loss": 1.3859, + "theoretical_loss": 3.451016607385185, + "tokens_seen": 1862205440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431919191919192, + "loss": 1.2745, + "theoretical_loss": 3.451006338635517, + "tokens_seen": 1862270976 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044317171717171716, + "loss": 1.264, + "theoretical_loss": 3.4509960703483946, + "tokens_seen": 1862336512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431515151515151, + "loss": 1.4957, + "theoretical_loss": 3.450985802523782, + "tokens_seen": 1862402048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431313131313132, + "loss": 1.1714, + "theoretical_loss": 3.45097553516164, + "tokens_seen": 1862467584 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044311111111111114, + "loss": 1.6347, + "theoretical_loss": 3.4509652682619327, + "tokens_seen": 1862533120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004430909090909091, + "loss": 1.456, + "theoretical_loss": 3.4509550018246236, + "tokens_seen": 1862598656 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044307070707070705, + "loss": 1.48, + "theoretical_loss": 3.4509447358496743, + "tokens_seen": 1862664192 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044305050505050506, + "loss": 1.5629, + "theoretical_loss": 3.450934470337048, + "tokens_seen": 1862729728 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044303030303030307, + "loss": 1.2863, + "theoretical_loss": 3.450924205286708, + "tokens_seen": 1862795264 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044301010101010103, + "loss": 1.5154, + "theoretical_loss": 3.4509139406986176, + "tokens_seen": 1862860800 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442989898989899, + "loss": 1.4515, + "theoretical_loss": 3.450903676572739, + "tokens_seen": 1862926336 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044296969696969694, + "loss": 1.3576, + "theoretical_loss": 3.4508934129090356, + "tokens_seen": 1862991872 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044294949494949495, + "loss": 1.4711, + "theoretical_loss": 3.4508831497074697, + "tokens_seen": 1863057408 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.35129520297050476, + "objective/train/avg_token_score": 0.03377072885632515, + "objective/train/docs_used": 1053498, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3309406042099, + "objective/train/original_loss": 4.976852893829346, + "objective/train/theoretical_loss": 3.4508728869680043, + "objective/train/tokens_used": 221982176, + "objective/train/unlikelihood_loss": 0.04581369832158089, + "theoretical_loss": 3.4508728869680043, + "tokens_seen": 1863122944 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044292929292929296, + "loss": 1.6527, + "theoretical_loss": 3.4508728869680043, + "tokens_seen": 1863122944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429090909090909, + "loss": 1.2831, + "theoretical_loss": 3.450862624690603, + "tokens_seen": 1863188480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428888888888889, + "loss": 1.2713, + "theoretical_loss": 3.450852362875229, + "tokens_seen": 1863254016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428686868686869, + "loss": 1.5471, + "theoretical_loss": 3.450842101521844, + "tokens_seen": 1863319552 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044284848484848485, + "loss": 1.3051, + "theoretical_loss": 3.4508318406304115, + "tokens_seen": 1863385088 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044282828282828286, + "loss": 1.401, + "theoretical_loss": 3.4508215802008952, + "tokens_seen": 1863450624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428080808080808, + "loss": 1.2803, + "theoretical_loss": 3.450811320233257, + "tokens_seen": 1863516160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044278787878787877, + "loss": 1.3002, + "theoretical_loss": 3.4508010607274606, + "tokens_seen": 1863581696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427676767676768, + "loss": 1.478, + "theoretical_loss": 3.4507908016834685, + "tokens_seen": 1863647232 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044274747474747474, + "loss": 1.4375, + "theoretical_loss": 3.4507805431012435, + "tokens_seen": 1863712768 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044272727272727275, + "loss": 1.3306, + "theoretical_loss": 3.4507702849807496, + "tokens_seen": 1863778304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427070707070707, + "loss": 1.4733, + "theoretical_loss": 3.450760027321948, + "tokens_seen": 1863843840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426868686868687, + "loss": 1.5835, + "theoretical_loss": 3.450749770124804, + "tokens_seen": 1863909376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426666666666667, + "loss": 1.4993, + "theoretical_loss": 3.4507395133892786, + "tokens_seen": 1863974912 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044264646464646463, + "loss": 1.423, + "theoretical_loss": 3.450729257115336, + "tokens_seen": 1864040448 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044262626262626264, + "loss": 1.493, + "theoretical_loss": 3.4507190013029385, + "tokens_seen": 1864105984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426060606060606, + "loss": 1.5426, + "theoretical_loss": 3.45070874595205, + "tokens_seen": 1864171520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425858585858586, + "loss": 1.5896, + "theoretical_loss": 3.4506984910626324, + "tokens_seen": 1864237056 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044256565656565657, + "loss": 1.5533, + "theoretical_loss": 3.450688236634649, + "tokens_seen": 1864302592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425454545454545, + "loss": 1.2419, + "theoretical_loss": 3.450677982668063, + "tokens_seen": 1864368128 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044252525252525254, + "loss": 1.2114, + "theoretical_loss": 3.450667729162838, + "tokens_seen": 1864433664 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044250505050505055, + "loss": 1.3821, + "theoretical_loss": 3.450657476118936, + "tokens_seen": 1864499200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424848484848485, + "loss": 1.4739, + "theoretical_loss": 3.450647223536321, + "tokens_seen": 1864564736 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044246464646464646, + "loss": 1.5295, + "theoretical_loss": 3.450636971414955, + "tokens_seen": 1864630272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424444444444444, + "loss": 1.2789, + "theoretical_loss": 3.4506267197548017, + "tokens_seen": 1864695808 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.39766618609428406, + "objective/train/avg_token_score": 0.03243307024240494, + "objective/train/docs_used": 1054143, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4144164323806763, + "objective/train/original_loss": 5.110235691070557, + "objective/train/theoretical_loss": 3.4506164685558245, + "objective/train/tokens_used": 223620576, + "objective/train/unlikelihood_loss": 0.04839473217725754, + "theoretical_loss": 3.4506164685558245, + "tokens_seen": 1864761344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044242424242424243, + "loss": 1.4935, + "theoretical_loss": 3.4506164685558245, + "tokens_seen": 1864761344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044240404040404044, + "loss": 1.3372, + "theoretical_loss": 3.4506062178179855, + "tokens_seen": 1864826880 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423838383838384, + "loss": 1.4895, + "theoretical_loss": 3.450595967541248, + "tokens_seen": 1864892416 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044236363636363635, + "loss": 1.3013, + "theoretical_loss": 3.450585717725576, + "tokens_seen": 1864957952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423434343434343, + "loss": 1.5954, + "theoretical_loss": 3.450575468370931, + "tokens_seen": 1865023488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423232323232324, + "loss": 1.5314, + "theoretical_loss": 3.4505652194772773, + "tokens_seen": 1865089024 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044230303030303033, + "loss": 1.386, + "theoretical_loss": 3.4505549710445775, + "tokens_seen": 1865154560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422828282828283, + "loss": 1.3209, + "theoretical_loss": 3.450544723072795, + "tokens_seen": 1865220096 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044226262626262625, + "loss": 1.5006, + "theoretical_loss": 3.4505344755618923, + "tokens_seen": 1865285632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422424242424242, + "loss": 1.6062, + "theoretical_loss": 3.450524228511833, + "tokens_seen": 1865351168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044222222222222227, + "loss": 1.4058, + "theoretical_loss": 3.4505139819225796, + "tokens_seen": 1865416704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422020202020202, + "loss": 1.5056, + "theoretical_loss": 3.450503735794096, + "tokens_seen": 1865482240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421818181818182, + "loss": 1.4215, + "theoretical_loss": 3.4504934901263447, + "tokens_seen": 1865547776 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044216161616161614, + "loss": 1.2773, + "theoretical_loss": 3.450483244919289, + "tokens_seen": 1865613312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421414141414142, + "loss": 1.7203, + "theoretical_loss": 3.450473000172892, + "tokens_seen": 1865678848 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044212121212121216, + "loss": 1.412, + "theoretical_loss": 3.450462755887117, + "tokens_seen": 1865744384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421010101010101, + "loss": 1.3453, + "theoretical_loss": 3.4504525120619265, + "tokens_seen": 1865809920 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420808080808081, + "loss": 1.4669, + "theoretical_loss": 3.4504422686972838, + "tokens_seen": 1865875456 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044206060606060603, + "loss": 1.4215, + "theoretical_loss": 3.4504320257931527, + "tokens_seen": 1865940992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420404040404041, + "loss": 1.5517, + "theoretical_loss": 3.450421783349496, + "tokens_seen": 1866006528 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044202020202020205, + "loss": 1.2322, + "theoretical_loss": 3.450411541366276, + "tokens_seen": 1866072064 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442, + "loss": 1.578, + "theoretical_loss": 3.450401299843457, + "tokens_seen": 1866137600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044197979797979797, + "loss": 1.392, + "theoretical_loss": 3.450391058781001, + "tokens_seen": 1866203136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004419595959595959, + "loss": 1.3039, + "theoretical_loss": 3.450380818178872, + "tokens_seen": 1866268672 + }, + { + "epoch": 0.12, + "learning_rate": 0.000441939393939394, + "loss": 1.6445, + "theoretical_loss": 3.4503705780370337, + "tokens_seen": 1866334208 + }, + { + "epoch": 0.12, + "objective/train/aligned_token_count": 0.35972630977630615, + "objective/train/avg_token_score": 0.007249823771417141, + "objective/train/docs_used": 1055457, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1949776411056519, + "objective/train/original_loss": 4.569217681884766, + "objective/train/theoretical_loss": 3.4503603383554475, + "objective/train/tokens_used": 225258976, + "objective/train/unlikelihood_loss": 0.057365208864212036, + "theoretical_loss": 3.4503603383554475, + "tokens_seen": 1866399744 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044191919191919195, + "loss": 1.4593, + "theoretical_loss": 3.4503603383554475, + "tokens_seen": 1866399744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418989898989899, + "loss": 1.3262, + "theoretical_loss": 3.450350099134078, + "tokens_seen": 1866465280 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044187878787878786, + "loss": 1.3928, + "theoretical_loss": 3.450339860372888, + "tokens_seen": 1866530816 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044185858585858587, + "loss": 1.4387, + "theoretical_loss": 3.4503296220718402, + "tokens_seen": 1866596352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418383838383839, + "loss": 1.4918, + "theoretical_loss": 3.4503193842308986, + "tokens_seen": 1866661888 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044181818181818184, + "loss": 1.2656, + "theoretical_loss": 3.4503091468500253, + "tokens_seen": 1866727424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417979797979798, + "loss": 1.4871, + "theoretical_loss": 3.4502989099291845, + "tokens_seen": 1866792960 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044177777777777775, + "loss": 1.5025, + "theoretical_loss": 3.450288673468339, + "tokens_seen": 1866858496 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044175757575757576, + "loss": 1.3684, + "theoretical_loss": 3.4502784374674516, + "tokens_seen": 1866924032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417373737373738, + "loss": 1.2633, + "theoretical_loss": 3.4502682019264856, + "tokens_seen": 1866989568 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044171717171717173, + "loss": 1.432, + "theoretical_loss": 3.450257966845405, + "tokens_seen": 1867055104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416969696969697, + "loss": 1.385, + "theoretical_loss": 3.450247732224172, + "tokens_seen": 1867120640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416767676767677, + "loss": 1.3333, + "theoretical_loss": 3.4502374980627506, + "tokens_seen": 1867186176 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044165656565656566, + "loss": 1.5311, + "theoretical_loss": 3.4502272643611027, + "tokens_seen": 1867251712 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044163636363636367, + "loss": 1.2273, + "theoretical_loss": 3.4502170311191933, + "tokens_seen": 1867317248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416161616161616, + "loss": 1.3906, + "theoretical_loss": 3.4502067983369846, + "tokens_seen": 1867382784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415959595959596, + "loss": 1.4789, + "theoretical_loss": 3.45019656601444, + "tokens_seen": 1867448320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415757575757576, + "loss": 1.4171, + "theoretical_loss": 3.450186334151522, + "tokens_seen": 1867513856 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044155555555555555, + "loss": 1.6178, + "theoretical_loss": 3.4501761027481948, + "tokens_seen": 1867579392 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044153535353535356, + "loss": 1.3581, + "theoretical_loss": 3.4501658718044217, + "tokens_seen": 1867644928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415151515151515, + "loss": 1.2223, + "theoretical_loss": 3.4501556413201646, + "tokens_seen": 1867710464 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044149494949494953, + "loss": 1.1382, + "theoretical_loss": 3.450145411295389, + "tokens_seen": 1867776000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414747474747475, + "loss": 1.4542, + "theoretical_loss": 3.450135181730056, + "tokens_seen": 1867841536 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044145454545454544, + "loss": 1.5126, + "theoretical_loss": 3.4501249526241295, + "tokens_seen": 1867907072 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044143434343434345, + "loss": 1.5464, + "theoretical_loss": 3.450114723977573, + "tokens_seen": 1867972608 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.5038489699363708, + "objective/train/avg_token_score": 0.0115775465965271, + "objective/train/docs_used": 1056292, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7018463611602783, + "objective/train/original_loss": 4.302921295166016, + "objective/train/theoretical_loss": 3.4501044957903506, + "objective/train/tokens_used": 226897376, + "objective/train/unlikelihood_loss": 0.07719095051288605, + "theoretical_loss": 3.4501044957903506, + "tokens_seen": 1868038144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414141414141414, + "loss": 1.5629, + "theoretical_loss": 3.4501044957903506, + "tokens_seen": 1868038144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413939393939394, + "loss": 1.3905, + "theoretical_loss": 3.450094268062424, + "tokens_seen": 1868103680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413737373737374, + "loss": 1.514, + "theoretical_loss": 3.4500840407937567, + "tokens_seen": 1868169216 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044135353535353533, + "loss": 1.4503, + "theoretical_loss": 3.450073813984313, + "tokens_seen": 1868234752 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044133333333333335, + "loss": 1.3241, + "theoretical_loss": 3.4500635876340553, + "tokens_seen": 1868300288 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044131313131313136, + "loss": 1.3679, + "theoretical_loss": 3.450053361742947, + "tokens_seen": 1868365824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412929292929293, + "loss": 1.3124, + "theoretical_loss": 3.450043136310952, + "tokens_seen": 1868431360 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044127272727272727, + "loss": 1.4361, + "theoretical_loss": 3.450032911338032, + "tokens_seen": 1868496896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412525252525252, + "loss": 1.5088, + "theoretical_loss": 3.4500226868241524, + "tokens_seen": 1868562432 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044123232323232324, + "loss": 1.5715, + "theoretical_loss": 3.450012462769275, + "tokens_seen": 1868627968 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044121212121212125, + "loss": 1.3653, + "theoretical_loss": 3.450002239173364, + "tokens_seen": 1868693504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411919191919192, + "loss": 1.6154, + "theoretical_loss": 3.449992016036382, + "tokens_seen": 1868759040 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044117171717171716, + "loss": 1.5006, + "theoretical_loss": 3.449981793358292, + "tokens_seen": 1868824576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411515151515151, + "loss": 1.5315, + "theoretical_loss": 3.4499715711390584, + "tokens_seen": 1868890112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411313131313132, + "loss": 1.416, + "theoretical_loss": 3.4499613493786443, + "tokens_seen": 1868955648 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044111111111111114, + "loss": 1.7077, + "theoretical_loss": 3.4499511280770125, + "tokens_seen": 1869021184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410909090909091, + "loss": 1.4733, + "theoretical_loss": 3.4499409072341263, + "tokens_seen": 1869086720 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044107070707070706, + "loss": 1.2313, + "theoretical_loss": 3.4499306868499495, + "tokens_seen": 1869152256 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441050505050505, + "loss": 1.3513, + "theoretical_loss": 3.449920466924445, + "tokens_seen": 1869217792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410303030303031, + "loss": 1.4508, + "theoretical_loss": 3.4499102474575762, + "tokens_seen": 1869283328 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044101010101010103, + "loss": 1.4852, + "theoretical_loss": 3.4499000284493073, + "tokens_seen": 1869348864 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440989898989899, + "loss": 1.4368, + "theoretical_loss": 3.4498898098996, + "tokens_seen": 1869414400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044096969696969695, + "loss": 1.3618, + "theoretical_loss": 3.4498795918084193, + "tokens_seen": 1869479936 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044094949494949496, + "loss": 1.3093, + "theoretical_loss": 3.4498693741757274, + "tokens_seen": 1869545472 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044092929292929297, + "loss": 1.3623, + "theoretical_loss": 3.4498591570014883, + "tokens_seen": 1869611008 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.49095797538757324, + "objective/train/avg_token_score": 0.0304701067507267, + "objective/train/docs_used": 1057648, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6677335500717163, + "objective/train/original_loss": 4.48096227645874, + "objective/train/theoretical_loss": 3.4498489402856647, + "objective/train/tokens_used": 228535776, + "objective/train/unlikelihood_loss": 0.04762193188071251, + "theoretical_loss": 3.4498489402856647, + "tokens_seen": 1869676544 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044090909090909093, + "loss": 1.5815, + "theoretical_loss": 3.4498489402856647, + "tokens_seen": 1869676544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408888888888889, + "loss": 1.3001, + "theoretical_loss": 3.4498387240282207, + "tokens_seen": 1869742080 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044086868686868684, + "loss": 1.6185, + "theoretical_loss": 3.4498285082291194, + "tokens_seen": 1869807616 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044084848484848485, + "loss": 1.5549, + "theoretical_loss": 3.449818292888324, + "tokens_seen": 1869873152 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044082828282828286, + "loss": 1.3115, + "theoretical_loss": 3.4498080780057983, + "tokens_seen": 1869938688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408080808080808, + "loss": 1.3658, + "theoretical_loss": 3.449797863581505, + "tokens_seen": 1870004224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407878787878788, + "loss": 1.5948, + "theoretical_loss": 3.4497876496154083, + "tokens_seen": 1870069760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407676767676768, + "loss": 1.399, + "theoretical_loss": 3.4497774361074707, + "tokens_seen": 1870135296 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044074747474747474, + "loss": 1.3492, + "theoretical_loss": 3.4497672230576564, + "tokens_seen": 1870200832 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044072727272727276, + "loss": 1.3206, + "theoretical_loss": 3.4497570104659285, + "tokens_seen": 1870266368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407070707070707, + "loss": 1.6108, + "theoretical_loss": 3.4497467983322503, + "tokens_seen": 1870331904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044068686868686867, + "loss": 1.559, + "theoretical_loss": 3.4497365866565852, + "tokens_seen": 1870397440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406666666666667, + "loss": 1.3866, + "theoretical_loss": 3.4497263754388965, + "tokens_seen": 1870462976 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044064646464646464, + "loss": 1.2945, + "theoretical_loss": 3.449716164679148, + "tokens_seen": 1870528512 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044062626262626265, + "loss": 1.5109, + "theoretical_loss": 3.449705954377303, + "tokens_seen": 1870594048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406060606060606, + "loss": 1.4244, + "theoretical_loss": 3.449695744533325, + "tokens_seen": 1870659584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405858585858586, + "loss": 1.4769, + "theoretical_loss": 3.449685535147177, + "tokens_seen": 1870725120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405656565656566, + "loss": 1.5157, + "theoretical_loss": 3.4496753262188227, + "tokens_seen": 1870790656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044054545454545453, + "loss": 1.2994, + "theoretical_loss": 3.4496651177482254, + "tokens_seen": 1870856192 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044052525252525254, + "loss": 1.4714, + "theoretical_loss": 3.449654909735349, + "tokens_seen": 1870921728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405050505050505, + "loss": 1.3804, + "theoretical_loss": 3.449644702180156, + "tokens_seen": 1870987264 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404848484848485, + "loss": 1.4414, + "theoretical_loss": 3.4496344950826114, + "tokens_seen": 1871052800 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044046464646464647, + "loss": 1.3298, + "theoretical_loss": 3.4496242884426773, + "tokens_seen": 1871118336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404444444444445, + "loss": 1.3629, + "theoretical_loss": 3.449614082260317, + "tokens_seen": 1871183872 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044042424242424243, + "loss": 1.442, + "theoretical_loss": 3.4496038765354955, + "tokens_seen": 1871249408 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.4029814302921295, + "objective/train/avg_token_score": 0.022246306762099266, + "objective/train/docs_used": 1058498, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2657883167266846, + "objective/train/original_loss": 4.2054595947265625, + "objective/train/theoretical_loss": 3.449593671268175, + "objective/train/tokens_used": 230174176, + "objective/train/unlikelihood_loss": 0.0596332885324955, + "theoretical_loss": 3.449593671268175, + "tokens_seen": 1871314944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044040404040404044, + "loss": 1.2121, + "theoretical_loss": 3.449593671268175, + "tokens_seen": 1871314944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403838383838384, + "loss": 1.4529, + "theoretical_loss": 3.449583466458319, + "tokens_seen": 1871380480 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044036363636363636, + "loss": 1.3306, + "theoretical_loss": 3.4495732621058917, + "tokens_seen": 1871446016 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044034343434343437, + "loss": 1.7467, + "theoretical_loss": 3.449563058210856, + "tokens_seen": 1871511552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403232323232323, + "loss": 1.2752, + "theoretical_loss": 3.4495528547731755, + "tokens_seen": 1871577088 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044030303030303034, + "loss": 1.337, + "theoretical_loss": 3.4495426517928136, + "tokens_seen": 1871642624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402828282828283, + "loss": 1.2836, + "theoretical_loss": 3.449532449269734, + "tokens_seen": 1871708160 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044026262626262625, + "loss": 1.3727, + "theoretical_loss": 3.4495222472039, + "tokens_seen": 1871773696 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044024242424242426, + "loss": 1.3912, + "theoretical_loss": 3.449512045595275, + "tokens_seen": 1871839232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402222222222222, + "loss": 1.4787, + "theoretical_loss": 3.449501844443823, + "tokens_seen": 1871904768 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044020202020202023, + "loss": 1.5051, + "theoretical_loss": 3.449491643749507, + "tokens_seen": 1871970304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401818181818182, + "loss": 1.6093, + "theoretical_loss": 3.449481443512291, + "tokens_seen": 1872035840 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044016161616161614, + "loss": 1.4234, + "theoretical_loss": 3.449471243732138, + "tokens_seen": 1872101376 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044014141414141416, + "loss": 1.6637, + "theoretical_loss": 3.449461044409012, + "tokens_seen": 1872166912 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044012121212121217, + "loss": 1.4772, + "theoretical_loss": 3.449450845542876, + "tokens_seen": 1872232448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401010101010101, + "loss": 1.491, + "theoretical_loss": 3.4494406471336934, + "tokens_seen": 1872297984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400808080808081, + "loss": 1.6623, + "theoretical_loss": 3.4494304491814285, + "tokens_seen": 1872363520 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044006060606060604, + "loss": 1.5462, + "theoretical_loss": 3.449420251686045, + "tokens_seen": 1872429056 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044004040404040405, + "loss": 1.4287, + "theoretical_loss": 3.4494100546475055, + "tokens_seen": 1872494592 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044002020202020206, + "loss": 1.532, + "theoretical_loss": 3.4493998580657736, + "tokens_seen": 1872560128 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044, + "loss": 1.2202, + "theoretical_loss": 3.4493896619408138, + "tokens_seen": 1872625664 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043997979797979797, + "loss": 1.4907, + "theoretical_loss": 3.449379466272589, + "tokens_seen": 1872691200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043995959595959593, + "loss": 1.61, + "theoretical_loss": 3.4493692710610624, + "tokens_seen": 1872756736 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439939393939394, + "loss": 1.4321, + "theoretical_loss": 3.449359076306198, + "tokens_seen": 1872822272 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043991919191919195, + "loss": 1.608, + "theoretical_loss": 3.4493488820079596, + "tokens_seen": 1872887808 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.39846041798591614, + "objective/train/avg_token_score": 0.02917778491973877, + "objective/train/docs_used": 1059136, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3530430793762207, + "objective/train/original_loss": 4.573091506958008, + "objective/train/theoretical_loss": 3.4493386881663106, + "objective/train/tokens_used": 231812576, + "objective/train/unlikelihood_loss": 0.08011636883020401, + "theoretical_loss": 3.4493386881663106, + "tokens_seen": 1872953344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398989898989899, + "loss": 1.4504, + "theoretical_loss": 3.4493386881663106, + "tokens_seen": 1872953344 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043987878787878787, + "loss": 1.2369, + "theoretical_loss": 3.4493284947812146, + "tokens_seen": 1873018880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398585858585858, + "loss": 1.5826, + "theoretical_loss": 3.4493183018526348, + "tokens_seen": 1873084416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398383838383839, + "loss": 1.2305, + "theoretical_loss": 3.449308109380535, + "tokens_seen": 1873149952 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043981818181818184, + "loss": 1.5691, + "theoretical_loss": 3.449297917364879, + "tokens_seen": 1873215488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397979797979798, + "loss": 1.5117, + "theoretical_loss": 3.44928772580563, + "tokens_seen": 1873281024 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043977777777777776, + "loss": 1.4064, + "theoretical_loss": 3.449277534702752, + "tokens_seen": 1873346560 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043975757575757577, + "loss": 1.6067, + "theoretical_loss": 3.4492673440562083, + "tokens_seen": 1873412096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397373737373738, + "loss": 1.4002, + "theoretical_loss": 3.449257153865963, + "tokens_seen": 1873477632 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043971717171717174, + "loss": 1.6485, + "theoretical_loss": 3.449246964131979, + "tokens_seen": 1873543168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396969696969697, + "loss": 1.4152, + "theoretical_loss": 3.4492367748542203, + "tokens_seen": 1873608704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043967676767676765, + "loss": 1.572, + "theoretical_loss": 3.4492265860326508, + "tokens_seen": 1873674240 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043965656565656566, + "loss": 1.545, + "theoretical_loss": 3.449216397667233, + "tokens_seen": 1873739776 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043963636363636367, + "loss": 1.3972, + "theoretical_loss": 3.449206209757932, + "tokens_seen": 1873805312 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043961616161616163, + "loss": 1.3669, + "theoretical_loss": 3.4491960223047107, + "tokens_seen": 1873870848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395959595959596, + "loss": 1.3002, + "theoretical_loss": 3.4491858353075324, + "tokens_seen": 1873936384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395757575757576, + "loss": 1.4357, + "theoretical_loss": 3.449175648766362, + "tokens_seen": 1874001920 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043955555555555555, + "loss": 1.432, + "theoretical_loss": 3.449165462681161, + "tokens_seen": 1874067456 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043953535353535357, + "loss": 1.3794, + "theoretical_loss": 3.4491552770518954, + "tokens_seen": 1874132992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395151515151515, + "loss": 1.5028, + "theoretical_loss": 3.449145091878527, + "tokens_seen": 1874198528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394949494949495, + "loss": 1.5145, + "theoretical_loss": 3.4491349071610204, + "tokens_seen": 1874264064 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394747474747475, + "loss": 1.3535, + "theoretical_loss": 3.449124722899339, + "tokens_seen": 1874329600 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043945454545454545, + "loss": 1.4699, + "theoretical_loss": 3.449114539093447, + "tokens_seen": 1874395136 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043943434343434346, + "loss": 1.3388, + "theoretical_loss": 3.449104355743307, + "tokens_seen": 1874460672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394141414141414, + "loss": 1.2571, + "theoretical_loss": 3.4490941728488838, + "tokens_seen": 1874526208 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.3365102708339691, + "objective/train/avg_token_score": 0.021979324519634247, + "objective/train/docs_used": 1059915, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1313660144805908, + "objective/train/original_loss": 4.815589904785156, + "objective/train/theoretical_loss": 3.44908399041014, + "objective/train/tokens_used": 233450976, + "objective/train/unlikelihood_loss": 0.04281238839030266, + "theoretical_loss": 3.44908399041014, + "tokens_seen": 1874591744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393939393939394, + "loss": 1.4928, + "theoretical_loss": 3.44908399041014, + "tokens_seen": 1874591744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393737373737374, + "loss": 1.3874, + "theoretical_loss": 3.4490738084270403, + "tokens_seen": 1874657280 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043935353535353534, + "loss": 1.4704, + "theoretical_loss": 3.4490636268995476, + "tokens_seen": 1874722816 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043933333333333335, + "loss": 1.5677, + "theoretical_loss": 3.4490534458276256, + "tokens_seen": 1874788352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393131313131313, + "loss": 1.499, + "theoretical_loss": 3.449043265211239, + "tokens_seen": 1874853888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392929292929293, + "loss": 1.5894, + "theoretical_loss": 3.4490330850503503, + "tokens_seen": 1874919424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392727272727273, + "loss": 1.4306, + "theoretical_loss": 3.4490229053449237, + "tokens_seen": 1874984960 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043925252525252523, + "loss": 1.3304, + "theoretical_loss": 3.449012726094923, + "tokens_seen": 1875050496 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043923232323232324, + "loss": 1.5022, + "theoretical_loss": 3.449002547300312, + "tokens_seen": 1875116032 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043921212121212125, + "loss": 1.3642, + "theoretical_loss": 3.4489923689610538, + "tokens_seen": 1875181568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391919191919192, + "loss": 1.3564, + "theoretical_loss": 3.4489821910771123, + "tokens_seen": 1875247104 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043917171717171717, + "loss": 1.4577, + "theoretical_loss": 3.448972013648452, + "tokens_seen": 1875312640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391515151515151, + "loss": 1.3256, + "theoretical_loss": 3.4489618366750356, + "tokens_seen": 1875378176 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043913131313131314, + "loss": 1.3953, + "theoretical_loss": 3.448951660156828, + "tokens_seen": 1875443712 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043911111111111115, + "loss": 1.44, + "theoretical_loss": 3.4489414840937913, + "tokens_seen": 1875509248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390909090909091, + "loss": 1.3612, + "theoretical_loss": 3.448931308485891, + "tokens_seen": 1875574784 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043907070707070706, + "loss": 1.6487, + "theoretical_loss": 3.4489211333330894, + "tokens_seen": 1875640320 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439050505050505, + "loss": 1.4251, + "theoretical_loss": 3.4489109586353512, + "tokens_seen": 1875705856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390303030303031, + "loss": 1.2871, + "theoretical_loss": 3.44890078439264, + "tokens_seen": 1875771392 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043901010101010104, + "loss": 1.5798, + "theoretical_loss": 3.448890610604918, + "tokens_seen": 1875836928 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438989898989899, + "loss": 1.3093, + "theoretical_loss": 3.4488804372721518, + "tokens_seen": 1875902464 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043896969696969695, + "loss": 1.6002, + "theoretical_loss": 3.448870264394303, + "tokens_seen": 1875968000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389494949494949, + "loss": 1.5905, + "theoretical_loss": 3.448860091971336, + "tokens_seen": 1876033536 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438929292929293, + "loss": 1.3809, + "theoretical_loss": 3.448849920003215, + "tokens_seen": 1876099072 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043890909090909093, + "loss": 1.5747, + "theoretical_loss": 3.4488397484899034, + "tokens_seen": 1876164608 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.3816593289375305, + "objective/train/avg_token_score": 0.014148054644465446, + "objective/train/docs_used": 1060624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0186139345169067, + "objective/train/original_loss": 3.7407007217407227, + "objective/train/theoretical_loss": 3.448829577431365, + "objective/train/tokens_used": 235089376, + "objective/train/unlikelihood_loss": 0.09927164018154144, + "theoretical_loss": 3.448829577431365, + "tokens_seen": 1876230144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388888888888889, + "loss": 1.2479, + "theoretical_loss": 3.448829577431365, + "tokens_seen": 1876230144 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043886868686868685, + "loss": 1.4699, + "theoretical_loss": 3.448819406827563, + "tokens_seen": 1876295680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388484848484848, + "loss": 1.3708, + "theoretical_loss": 3.448809236678462, + "tokens_seen": 1876361216 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043882828282828287, + "loss": 1.4538, + "theoretical_loss": 3.4487990669840256, + "tokens_seen": 1876426752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388080808080808, + "loss": 1.7556, + "theoretical_loss": 3.448788897744217, + "tokens_seen": 1876492288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387878787878788, + "loss": 1.3391, + "theoretical_loss": 3.4487787289590015, + "tokens_seen": 1876557824 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043876767676767674, + "loss": 1.5276, + "theoretical_loss": 3.4487685606283414, + "tokens_seen": 1876623360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387474747474748, + "loss": 1.6076, + "theoretical_loss": 3.4487583927522008, + "tokens_seen": 1876688896 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043872727272727276, + "loss": 1.3362, + "theoretical_loss": 3.4487482253305437, + "tokens_seen": 1876754432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387070707070707, + "loss": 1.4142, + "theoretical_loss": 3.4487380583633342, + "tokens_seen": 1876819968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386868686868687, + "loss": 1.6026, + "theoretical_loss": 3.4487278918505364, + "tokens_seen": 1876885504 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043866666666666663, + "loss": 1.4723, + "theoretical_loss": 3.448717725792113, + "tokens_seen": 1876951040 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386464646464647, + "loss": 1.6469, + "theoretical_loss": 3.448707560188028, + "tokens_seen": 1877016576 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043862626262626265, + "loss": 1.5528, + "theoretical_loss": 3.4486973950382467, + "tokens_seen": 1877082112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386060606060606, + "loss": 1.4923, + "theoretical_loss": 3.448687230342731, + "tokens_seen": 1877147648 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043858585858585857, + "loss": 1.1962, + "theoretical_loss": 3.448677066101446, + "tokens_seen": 1877213184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385656565656566, + "loss": 1.3383, + "theoretical_loss": 3.448666902314355, + "tokens_seen": 1877278720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385454545454546, + "loss": 1.354, + "theoretical_loss": 3.4486567389814224, + "tokens_seen": 1877344256 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043852525252525255, + "loss": 1.4967, + "theoretical_loss": 3.4486465761026115, + "tokens_seen": 1877409792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385050505050505, + "loss": 1.362, + "theoretical_loss": 3.448636413677886, + "tokens_seen": 1877475328 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043848484848484846, + "loss": 1.5587, + "theoretical_loss": 3.4486262517072106, + "tokens_seen": 1877540864 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043846464646464647, + "loss": 1.4487, + "theoretical_loss": 3.4486160901905487, + "tokens_seen": 1877606400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384444444444445, + "loss": 1.242, + "theoretical_loss": 3.448605929127864, + "tokens_seen": 1877671936 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043842424242424244, + "loss": 1.5602, + "theoretical_loss": 3.4485957685191204, + "tokens_seen": 1877737472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384040404040404, + "loss": 1.3082, + "theoretical_loss": 3.448585608364282, + "tokens_seen": 1877803008 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.49346286058425903, + "objective/train/avg_token_score": 0.00607198616489768, + "objective/train/docs_used": 1062058, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6114000082015991, + "objective/train/original_loss": 3.4861137866973877, + "objective/train/theoretical_loss": 3.4485754486633127, + "objective/train/tokens_used": 236727776, + "objective/train/unlikelihood_loss": 0.09068997949361801, + "theoretical_loss": 3.4485754486633127, + "tokens_seen": 1877868544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383838383838384, + "loss": 1.4889, + "theoretical_loss": 3.4485754486633127, + "tokens_seen": 1877868544 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043836363636363636, + "loss": 1.4036, + "theoretical_loss": 3.448565289416176, + "tokens_seen": 1877934080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383434343434344, + "loss": 1.5098, + "theoretical_loss": 3.448555130622836, + "tokens_seen": 1877999616 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043832323232323233, + "loss": 1.429, + "theoretical_loss": 3.4485449722832566, + "tokens_seen": 1878065152 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383030303030303, + "loss": 1.2384, + "theoretical_loss": 3.4485348143974024, + "tokens_seen": 1878130688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382828282828283, + "loss": 1.3225, + "theoretical_loss": 3.448524656965236, + "tokens_seen": 1878196224 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043826262626262626, + "loss": 1.5303, + "theoretical_loss": 3.448514499986722, + "tokens_seen": 1878261760 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043824242424242427, + "loss": 1.4846, + "theoretical_loss": 3.448504343461824, + "tokens_seen": 1878327296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382222222222222, + "loss": 1.4366, + "theoretical_loss": 3.448494187390507, + "tokens_seen": 1878392832 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043820202020202024, + "loss": 1.2521, + "theoretical_loss": 3.448484031772734, + "tokens_seen": 1878458368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381818181818182, + "loss": 1.2788, + "theoretical_loss": 3.4484738766084684, + "tokens_seen": 1878523904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043816161616161615, + "loss": 1.4862, + "theoretical_loss": 3.4484637218976752, + "tokens_seen": 1878589440 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043814141414141416, + "loss": 1.5265, + "theoretical_loss": 3.4484535676403176, + "tokens_seen": 1878654976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381212121212121, + "loss": 1.734, + "theoretical_loss": 3.44844341383636, + "tokens_seen": 1878720512 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043810101010101013, + "loss": 1.465, + "theoretical_loss": 3.448433260485766, + "tokens_seen": 1878786048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380808080808081, + "loss": 1.44, + "theoretical_loss": 3.4484231075885, + "tokens_seen": 1878851584 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043806060606060604, + "loss": 1.6416, + "theoretical_loss": 3.4484129551445255, + "tokens_seen": 1878917120 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043804040404040405, + "loss": 1.3597, + "theoretical_loss": 3.4484028031538063, + "tokens_seen": 1878982656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043802020202020206, + "loss": 1.7944, + "theoretical_loss": 3.4483926516163073, + "tokens_seen": 1879048192 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438, + "loss": 1.4988, + "theoretical_loss": 3.4483825005319915, + "tokens_seen": 1879113728 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437979797979798, + "loss": 1.2804, + "theoretical_loss": 3.448372349900823, + "tokens_seen": 1879179264 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043795959595959593, + "loss": 1.3335, + "theoretical_loss": 3.448362199722766, + "tokens_seen": 1879244800 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043793939393939395, + "loss": 1.5275, + "theoretical_loss": 3.4483520499977844, + "tokens_seen": 1879310336 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043791919191919196, + "loss": 1.4088, + "theoretical_loss": 3.4483419007258425, + "tokens_seen": 1879375872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004378989898989899, + "loss": 1.5582, + "theoretical_loss": 3.4483317519069043, + "tokens_seen": 1879441408 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.5173509120941162, + "objective/train/avg_token_score": 0.04670140892267227, + "objective/train/docs_used": 1062685, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7349787950515747, + "objective/train/original_loss": 3.844883918762207, + "objective/train/theoretical_loss": 3.4483216035409328, + "objective/train/tokens_used": 238366176, + "objective/train/unlikelihood_loss": 0.05952979996800423, + "theoretical_loss": 3.4483216035409328, + "tokens_seen": 1879506944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043787878787878787, + "loss": 1.3996, + "theoretical_loss": 3.4483216035409328, + "tokens_seen": 1879506944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043785858585858583, + "loss": 1.4732, + "theoretical_loss": 3.4483114556278927, + "tokens_seen": 1879572480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004378383838383839, + "loss": 1.5456, + "theoretical_loss": 3.448301308167748, + "tokens_seen": 1879638016 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043781818181818185, + "loss": 1.732, + "theoretical_loss": 3.448291161160463, + "tokens_seen": 1879703552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377979797979798, + "loss": 1.5276, + "theoretical_loss": 3.4482810146060014, + "tokens_seen": 1879769088 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043777777777777776, + "loss": 1.6654, + "theoretical_loss": 3.4482708685043266, + "tokens_seen": 1879834624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377575757575757, + "loss": 1.4529, + "theoretical_loss": 3.4482607228554034, + "tokens_seen": 1879900160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377373737373738, + "loss": 1.367, + "theoretical_loss": 3.448250577659196, + "tokens_seen": 1879965696 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043771717171717174, + "loss": 1.3748, + "theoretical_loss": 3.448240432915667, + "tokens_seen": 1880031232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376969696969697, + "loss": 1.3824, + "theoretical_loss": 3.4482302886247824, + "tokens_seen": 1880096768 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043767676767676766, + "loss": 1.4715, + "theoretical_loss": 3.448220144786505, + "tokens_seen": 1880162304 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043765656565656567, + "loss": 1.3517, + "theoretical_loss": 3.448210001400799, + "tokens_seen": 1880227840 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376363636363637, + "loss": 1.3135, + "theoretical_loss": 3.4481998584676288, + "tokens_seen": 1880293376 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043761616161616164, + "loss": 1.5466, + "theoretical_loss": 3.448189715986958, + "tokens_seen": 1880358912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375959595959596, + "loss": 1.5333, + "theoretical_loss": 3.4481795739587504, + "tokens_seen": 1880424448 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043757575757575755, + "loss": 1.3885, + "theoretical_loss": 3.448169432382971, + "tokens_seen": 1880489984 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043755555555555556, + "loss": 1.6391, + "theoretical_loss": 3.448159291259583, + "tokens_seen": 1880555520 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043753535353535357, + "loss": 1.7587, + "theoretical_loss": 3.448149150588551, + "tokens_seen": 1880621056 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043751515151515153, + "loss": 1.4384, + "theoretical_loss": 3.4481390103698386, + "tokens_seen": 1880686592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374949494949495, + "loss": 1.3425, + "theoretical_loss": 3.44812887060341, + "tokens_seen": 1880752128 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374747474747475, + "loss": 1.597, + "theoretical_loss": 3.448118731289229, + "tokens_seen": 1880817664 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043745454545454545, + "loss": 1.2963, + "theoretical_loss": 3.448108592427261, + "tokens_seen": 1880883200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043743434343434346, + "loss": 1.4757, + "theoretical_loss": 3.448098454017469, + "tokens_seen": 1880948736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374141414141414, + "loss": 1.3524, + "theoretical_loss": 3.4480883160598164, + "tokens_seen": 1881014272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373939393939394, + "loss": 1.672, + "theoretical_loss": 3.448078178554268, + "tokens_seen": 1881079808 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.41361191868782043, + "objective/train/avg_token_score": 0.008350033313035965, + "objective/train/docs_used": 1064316, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4659239053726196, + "objective/train/original_loss": 4.062493801116943, + "objective/train/theoretical_loss": 3.4480680415007887, + "objective/train/tokens_used": 240004576, + "objective/train/unlikelihood_loss": 0.06435056775808334, + "theoretical_loss": 3.4480680415007887, + "tokens_seen": 1881145344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373737373737374, + "loss": 1.3753, + "theoretical_loss": 3.4480680415007887, + "tokens_seen": 1881145344 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043735353535353535, + "loss": 1.301, + "theoretical_loss": 3.4480579048993416, + "tokens_seen": 1881210880 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043733333333333336, + "loss": 1.3698, + "theoretical_loss": 3.448047768749891, + "tokens_seen": 1881276416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373131313131313, + "loss": 1.6219, + "theoretical_loss": 3.448037633052401, + "tokens_seen": 1881341952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372929292929293, + "loss": 1.4685, + "theoretical_loss": 3.4480274978068355, + "tokens_seen": 1881407488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372727272727273, + "loss": 1.3703, + "theoretical_loss": 3.448017363013159, + "tokens_seen": 1881473024 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043725252525252524, + "loss": 1.3487, + "theoretical_loss": 3.4480072286713357, + "tokens_seen": 1881538560 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043723232323232325, + "loss": 1.1748, + "theoretical_loss": 3.4479970947813294, + "tokens_seen": 1881604096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372121212121212, + "loss": 1.4412, + "theoretical_loss": 3.447986961343104, + "tokens_seen": 1881669632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371919191919192, + "loss": 1.5357, + "theoretical_loss": 3.4479768283566243, + "tokens_seen": 1881735168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371717171717172, + "loss": 1.6326, + "theoretical_loss": 3.447966695821854, + "tokens_seen": 1881800704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043715151515151513, + "loss": 1.5148, + "theoretical_loss": 3.447956563738757, + "tokens_seen": 1881866240 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043713131313131314, + "loss": 1.6197, + "theoretical_loss": 3.4479464321072983, + "tokens_seen": 1881931776 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043711111111111115, + "loss": 1.3488, + "theoretical_loss": 3.4479363009274406, + "tokens_seen": 1881997312 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370909090909091, + "loss": 1.5204, + "theoretical_loss": 3.4479261701991497, + "tokens_seen": 1882062848 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043707070707070707, + "loss": 1.4517, + "theoretical_loss": 3.4479160399223887, + "tokens_seen": 1882128384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370505050505051, + "loss": 1.5259, + "theoretical_loss": 3.4479059100971217, + "tokens_seen": 1882193920 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043703030303030303, + "loss": 1.4554, + "theoretical_loss": 3.4478957807233135, + "tokens_seen": 1882259456 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043701010101010105, + "loss": 1.4525, + "theoretical_loss": 3.447885651800928, + "tokens_seen": 1882324992 + }, + { + "epoch": 0.13, + "learning_rate": 0.000436989898989899, + "loss": 1.2467, + "theoretical_loss": 3.4478755233299294, + "tokens_seen": 1882390528 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043696969696969696, + "loss": 1.3973, + "theoretical_loss": 3.4478653953102816, + "tokens_seen": 1882456064 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043694949494949497, + "loss": 1.3898, + "theoretical_loss": 3.4478552677419487, + "tokens_seen": 1882521600 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043692929292929293, + "loss": 1.2577, + "theoretical_loss": 3.4478451406248953, + "tokens_seen": 1882587136 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043690909090909094, + "loss": 1.4622, + "theoretical_loss": 3.4478350139590854, + "tokens_seen": 1882652672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004368888888888889, + "loss": 1.5288, + "theoretical_loss": 3.447824887744483, + "tokens_seen": 1882718208 + }, + { + "epoch": 0.13, + "objective/train/aligned_token_count": 0.41856059432029724, + "objective/train/avg_token_score": 0.06959132105112076, + "objective/train/docs_used": 1064917, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4428768157958984, + "objective/train/original_loss": 4.215625286102295, + "objective/train/theoretical_loss": 3.447814761981053, + "objective/train/tokens_used": 241642976, + "objective/train/unlikelihood_loss": 0.06671393662691116, + "theoretical_loss": 3.447814761981053, + "tokens_seen": 1882783744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043686868686868685, + "loss": 1.6306, + "theoretical_loss": 3.447814761981053, + "tokens_seen": 1882783744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043684848484848486, + "loss": 1.304, + "theoretical_loss": 3.447804636668759, + "tokens_seen": 1882849280 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004368282828282829, + "loss": 1.347, + "theoretical_loss": 3.4477945118075652, + "tokens_seen": 1882914816 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043680808080808083, + "loss": 1.4824, + "theoretical_loss": 3.4477843873974354, + "tokens_seen": 1882980352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367878787878788, + "loss": 1.3546, + "theoretical_loss": 3.447774263438335, + "tokens_seen": 1883045888 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043676767676767674, + "loss": 1.3092, + "theoretical_loss": 3.447764139930227, + "tokens_seen": 1883111424 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043674747474747476, + "loss": 1.3361, + "theoretical_loss": 3.4477540168730765, + "tokens_seen": 1883176960 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043672727272727277, + "loss": 1.2515, + "theoretical_loss": 3.4477438942668472, + "tokens_seen": 1883242496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367070707070707, + "loss": 1.3784, + "theoretical_loss": 3.447733772111503, + "tokens_seen": 1883308032 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366868686868687, + "loss": 1.4131, + "theoretical_loss": 3.447723650407009, + "tokens_seen": 1883373568 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043666666666666664, + "loss": 1.3234, + "theoretical_loss": 3.447713529153329, + "tokens_seen": 1883439104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366464646464647, + "loss": 1.3278, + "theoretical_loss": 3.447703408350427, + "tokens_seen": 1883504640 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043662626262626266, + "loss": 1.3996, + "theoretical_loss": 3.447693287998268, + "tokens_seen": 1883570176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366060606060606, + "loss": 1.3433, + "theoretical_loss": 3.447683168096815, + "tokens_seen": 1883635712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365858585858586, + "loss": 1.5963, + "theoretical_loss": 3.447673048646034, + "tokens_seen": 1883701248 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043656565656565653, + "loss": 1.5976, + "theoretical_loss": 3.4476629296458876, + "tokens_seen": 1883766784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365454545454546, + "loss": 1.4234, + "theoretical_loss": 3.447652811096341, + "tokens_seen": 1883832320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043652525252525255, + "loss": 1.4693, + "theoretical_loss": 3.447642692997358, + "tokens_seen": 1883897856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365050505050505, + "loss": 1.3536, + "theoretical_loss": 3.4476325753489028, + "tokens_seen": 1883963392 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043648484848484847, + "loss": 1.5021, + "theoretical_loss": 3.4476224581509403, + "tokens_seen": 1884028928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364646464646465, + "loss": 1.4861, + "theoretical_loss": 3.447612341403434, + "tokens_seen": 1884094464 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364444444444445, + "loss": 1.4886, + "theoretical_loss": 3.4476022251063485, + "tokens_seen": 1884160000 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043642424242424244, + "loss": 1.007, + "theoretical_loss": 3.447592109259648, + "tokens_seen": 1884225536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364040404040404, + "loss": 1.3627, + "theoretical_loss": 3.447581993863297, + "tokens_seen": 1884291072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043638383838383836, + "loss": 1.3705, + "theoretical_loss": 3.44757187891726, + "tokens_seen": 1884356608 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.40738025307655334, + "objective/train/avg_token_score": 0.015582275576889515, + "objective/train/docs_used": 1065907, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.418578028678894, + "objective/train/original_loss": 4.257914066314697, + "objective/train/theoretical_loss": 3.4475617644215006, + "objective/train/tokens_used": 243281376, + "objective/train/unlikelihood_loss": 0.0637265294790268, + "theoretical_loss": 3.4475617644215006, + "tokens_seen": 1884422144 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043636363636363637, + "loss": 1.5404, + "theoretical_loss": 3.4475617644215006, + "tokens_seen": 1884422144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363434343434344, + "loss": 1.2178, + "theoretical_loss": 3.447551650375984, + "tokens_seen": 1884487680 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043632323232323234, + "loss": 1.4998, + "theoretical_loss": 3.4475415367806734, + "tokens_seen": 1884553216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363030303030303, + "loss": 1.3251, + "theoretical_loss": 3.4475314236355343, + "tokens_seen": 1884618752 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362828282828283, + "loss": 1.4469, + "theoretical_loss": 3.44752131094053, + "tokens_seen": 1884684288 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043626262626262626, + "loss": 1.5319, + "theoretical_loss": 3.447511198695625, + "tokens_seen": 1884749824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362424242424243, + "loss": 1.3746, + "theoretical_loss": 3.4475010869007843, + "tokens_seen": 1884815360 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043622222222222223, + "loss": 1.2662, + "theoretical_loss": 3.4474909755559713, + "tokens_seen": 1884880896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362020202020202, + "loss": 1.4889, + "theoretical_loss": 3.4474808646611512, + "tokens_seen": 1884946432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361818181818182, + "loss": 1.2985, + "theoretical_loss": 3.4474707542162877, + "tokens_seen": 1885011968 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043616161616161616, + "loss": 1.4024, + "theoretical_loss": 3.447460644221345, + "tokens_seen": 1885077504 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043614141414141417, + "loss": 1.6368, + "theoretical_loss": 3.447450534676288, + "tokens_seen": 1885143040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361212121212121, + "loss": 1.4649, + "theoretical_loss": 3.447440425581081, + "tokens_seen": 1885208576 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043610101010101013, + "loss": 1.3478, + "theoretical_loss": 3.447430316935688, + "tokens_seen": 1885274112 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360808080808081, + "loss": 1.1866, + "theoretical_loss": 3.4474202087400734, + "tokens_seen": 1885339648 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043606060606060605, + "loss": 1.4361, + "theoretical_loss": 3.447410100994202, + "tokens_seen": 1885405184 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043604040404040406, + "loss": 1.5223, + "theoretical_loss": 3.4473999936980375, + "tokens_seen": 1885470720 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436020202020202, + "loss": 1.549, + "theoretical_loss": 3.4473898868515445, + "tokens_seen": 1885536256 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436, + "loss": 1.2515, + "theoretical_loss": 3.447379780454688, + "tokens_seen": 1885601792 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435979797979798, + "loss": 1.5817, + "theoretical_loss": 3.4473696745074314, + "tokens_seen": 1885667328 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043595959595959594, + "loss": 1.5611, + "theoretical_loss": 3.4473595690097394, + "tokens_seen": 1885732864 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043593939393939395, + "loss": 1.6062, + "theoretical_loss": 3.4473494639615767, + "tokens_seen": 1885798400 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043591919191919196, + "loss": 1.3081, + "theoretical_loss": 3.447339359362907, + "tokens_seen": 1885863936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358989898989899, + "loss": 1.5584, + "theoretical_loss": 3.447329255213696, + "tokens_seen": 1885929472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358787878787879, + "loss": 1.362, + "theoretical_loss": 3.4473191515139066, + "tokens_seen": 1885995008 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.4603494703769684, + "objective/train/avg_token_score": 0.004963160492479801, + "objective/train/docs_used": 1066425, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4913145303726196, + "objective/train/original_loss": 4.012363910675049, + "objective/train/theoretical_loss": 3.447309048263504, + "objective/train/tokens_used": 244919776, + "objective/train/unlikelihood_loss": 0.06790236383676529, + "theoretical_loss": 3.447309048263504, + "tokens_seen": 1886060544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043585858585858583, + "loss": 1.3731, + "theoretical_loss": 3.447309048263504, + "tokens_seen": 1886060544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043583838383838384, + "loss": 1.3272, + "theoretical_loss": 3.4472989454624523, + "tokens_seen": 1886126080 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043581818181818186, + "loss": 1.4449, + "theoretical_loss": 3.447288843110716, + "tokens_seen": 1886191616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357979797979798, + "loss": 1.4432, + "theoretical_loss": 3.4472787412082595, + "tokens_seen": 1886257152 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043577777777777777, + "loss": 1.5336, + "theoretical_loss": 3.4472686397550474, + "tokens_seen": 1886322688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357575757575757, + "loss": 1.4763, + "theoretical_loss": 3.4472585387510435, + "tokens_seen": 1886388224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357373737373738, + "loss": 1.3966, + "theoretical_loss": 3.447248438196213, + "tokens_seen": 1886453760 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043571717171717175, + "loss": 1.4625, + "theoretical_loss": 3.4472383380905205, + "tokens_seen": 1886519296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356969696969697, + "loss": 1.4471, + "theoretical_loss": 3.4472282384339294, + "tokens_seen": 1886584832 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043567676767676766, + "loss": 1.3535, + "theoretical_loss": 3.4472181392264045, + "tokens_seen": 1886650368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356565656565656, + "loss": 1.3392, + "theoretical_loss": 3.4472080404679106, + "tokens_seen": 1886715904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356363636363637, + "loss": 1.4189, + "theoretical_loss": 3.447197942158412, + "tokens_seen": 1886781440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043561616161616164, + "loss": 1.5119, + "theoretical_loss": 3.447187844297873, + "tokens_seen": 1886846976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355959595959596, + "loss": 1.6033, + "theoretical_loss": 3.447177746886258, + "tokens_seen": 1886912512 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043557575757575755, + "loss": 1.5703, + "theoretical_loss": 3.4471676499235318, + "tokens_seen": 1886978048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355555555555555, + "loss": 1.4541, + "theoretical_loss": 3.4471575534096583, + "tokens_seen": 1887043584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355353535353536, + "loss": 1.4322, + "theoretical_loss": 3.4471474573446024, + "tokens_seen": 1887109120 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043551515151515153, + "loss": 1.485, + "theoretical_loss": 3.4471373617283287, + "tokens_seen": 1887174656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354949494949495, + "loss": 1.4842, + "theoretical_loss": 3.447127266560801, + "tokens_seen": 1887240192 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043547474747474745, + "loss": 1.4015, + "theoretical_loss": 3.4471171718419846, + "tokens_seen": 1887305728 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043545454545454546, + "loss": 1.5789, + "theoretical_loss": 3.447107077571843, + "tokens_seen": 1887371264 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043543434343434347, + "loss": 1.5174, + "theoretical_loss": 3.447096983750342, + "tokens_seen": 1887436800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354141414141414, + "loss": 1.2054, + "theoretical_loss": 3.447086890377445, + "tokens_seen": 1887502336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353939393939394, + "loss": 1.3724, + "theoretical_loss": 3.447076797453117, + "tokens_seen": 1887567872 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043537373737373734, + "loss": 1.6524, + "theoretical_loss": 3.4470667049773214, + "tokens_seen": 1887633408 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.4772116243839264, + "objective/train/avg_token_score": 0.018777167424559593, + "objective/train/docs_used": 1067085, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5127924680709839, + "objective/train/original_loss": 4.050947666168213, + "objective/train/theoretical_loss": 3.4470566129500244, + "objective/train/tokens_used": 246558176, + "objective/train/unlikelihood_loss": 0.05088551715016365, + "theoretical_loss": 3.4470566129500244, + "tokens_seen": 1887698944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043535353535353535, + "loss": 1.3626, + "theoretical_loss": 3.4470566129500244, + "tokens_seen": 1887698944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043533333333333336, + "loss": 1.3871, + "theoretical_loss": 3.4470465213711896, + "tokens_seen": 1887764480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353131313131313, + "loss": 1.5453, + "theoretical_loss": 3.4470364302407814, + "tokens_seen": 1887830016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352929292929293, + "loss": 1.4407, + "theoretical_loss": 3.447026339558765, + "tokens_seen": 1887895552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352727272727273, + "loss": 1.4609, + "theoretical_loss": 3.4470162493251038, + "tokens_seen": 1887961088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352525252525253, + "loss": 1.4642, + "theoretical_loss": 3.4470061595397627, + "tokens_seen": 1888026624 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043523232323232325, + "loss": 1.2666, + "theoretical_loss": 3.446996070202707, + "tokens_seen": 1888092160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352121212121212, + "loss": 1.2846, + "theoretical_loss": 3.446985981313901, + "tokens_seen": 1888157696 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043519191919191917, + "loss": 1.4019, + "theoretical_loss": 3.4469758928733083, + "tokens_seen": 1888223232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351717171717172, + "loss": 1.4626, + "theoretical_loss": 3.446965804880894, + "tokens_seen": 1888288768 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351515151515152, + "loss": 1.1625, + "theoretical_loss": 3.446955717336623, + "tokens_seen": 1888354304 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043513131313131315, + "loss": 1.5753, + "theoretical_loss": 3.4469456302404597, + "tokens_seen": 1888419840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351111111111111, + "loss": 1.4806, + "theoretical_loss": 3.4469355435923683, + "tokens_seen": 1888485376 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350909090909091, + "loss": 1.5697, + "theoretical_loss": 3.4469254573923136, + "tokens_seen": 1888550912 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043507070707070707, + "loss": 1.4095, + "theoretical_loss": 3.4469153716402596, + "tokens_seen": 1888616448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350505050505051, + "loss": 1.5738, + "theoretical_loss": 3.4469052863361713, + "tokens_seen": 1888681984 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043503030303030304, + "loss": 1.4386, + "theoretical_loss": 3.4468952014800136, + "tokens_seen": 1888747520 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435010101010101, + "loss": 1.4282, + "theoretical_loss": 3.4468851170717505, + "tokens_seen": 1888813056 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434989898989899, + "loss": 1.4533, + "theoretical_loss": 3.446875033111347, + "tokens_seen": 1888878592 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043496969696969696, + "loss": 1.4092, + "theoretical_loss": 3.4468649495987678, + "tokens_seen": 1888944128 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434949494949495, + "loss": 1.4225, + "theoretical_loss": 3.4468548665339767, + "tokens_seen": 1889009664 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043492929292929293, + "loss": 1.1841, + "theoretical_loss": 3.446844783916939, + "tokens_seen": 1889075200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043490909090909094, + "loss": 1.4895, + "theoretical_loss": 3.4468347017476186, + "tokens_seen": 1889140736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348888888888889, + "loss": 1.4007, + "theoretical_loss": 3.446824620025981, + "tokens_seen": 1889206272 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043486868686868686, + "loss": 1.3433, + "theoretical_loss": 3.44681453875199, + "tokens_seen": 1889271808 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.4841764271259308, + "objective/train/avg_token_score": 0.00916412752121687, + "objective/train/docs_used": 1068281, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6944667100906372, + "objective/train/original_loss": 4.594786643981934, + "objective/train/theoretical_loss": 3.4468044579256105, + "objective/train/tokens_used": 248196576, + "objective/train/unlikelihood_loss": 0.028151439502835274, + "theoretical_loss": 3.4468044579256105, + "tokens_seen": 1889337344 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043484848484848487, + "loss": 1.437, + "theoretical_loss": 3.4468044579256105, + "tokens_seen": 1889337344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348282828282828, + "loss": 1.4151, + "theoretical_loss": 3.446794377546807, + "tokens_seen": 1889402880 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043480808080808084, + "loss": 1.3273, + "theoretical_loss": 3.4467842976155447, + "tokens_seen": 1889468416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347878787878788, + "loss": 1.5779, + "theoretical_loss": 3.4467742181317873, + "tokens_seen": 1889533952 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043476767676767675, + "loss": 1.5525, + "theoretical_loss": 3.4467641390955, + "tokens_seen": 1889599488 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043474747474747476, + "loss": 1.4035, + "theoretical_loss": 3.446754060506647, + "tokens_seen": 1889665024 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043472727272727277, + "loss": 1.4794, + "theoretical_loss": 3.4467439823651933, + "tokens_seen": 1889730560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043470707070707073, + "loss": 1.2579, + "theoretical_loss": 3.4467339046711034, + "tokens_seen": 1889796096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346868686868687, + "loss": 1.3739, + "theoretical_loss": 3.446723827424342, + "tokens_seen": 1889861632 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043466666666666664, + "loss": 1.4137, + "theoretical_loss": 3.4467137506248733, + "tokens_seen": 1889927168 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043464646464646465, + "loss": 1.4186, + "theoretical_loss": 3.446703674272663, + "tokens_seen": 1889992704 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043462626262626267, + "loss": 1.4476, + "theoretical_loss": 3.4466935983676743, + "tokens_seen": 1890058240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346060606060606, + "loss": 1.5298, + "theoretical_loss": 3.446683522909873, + "tokens_seen": 1890123776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345858585858586, + "loss": 1.4053, + "theoretical_loss": 3.446673447899223, + "tokens_seen": 1890189312 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043456565656565654, + "loss": 1.4032, + "theoretical_loss": 3.4466633733356895, + "tokens_seen": 1890254848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345454545454546, + "loss": 1.488, + "theoretical_loss": 3.4466532992192365, + "tokens_seen": 1890320384 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043452525252525256, + "loss": 1.3714, + "theoretical_loss": 3.4466432255498294, + "tokens_seen": 1890385920 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345050505050505, + "loss": 1.2779, + "theoretical_loss": 3.4466331523274327, + "tokens_seen": 1890451456 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043448484848484847, + "loss": 1.6398, + "theoretical_loss": 3.446623079552011, + "tokens_seen": 1890516992 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043446464646464643, + "loss": 1.319, + "theoretical_loss": 3.4466130072235286, + "tokens_seen": 1890582528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344444444444445, + "loss": 1.5699, + "theoretical_loss": 3.4466029353419505, + "tokens_seen": 1890648064 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043442424242424245, + "loss": 1.4971, + "theoretical_loss": 3.4465928639072416, + "tokens_seen": 1890713600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344040404040404, + "loss": 1.5592, + "theoretical_loss": 3.446582792919366, + "tokens_seen": 1890779136 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043438383838383836, + "loss": 1.4512, + "theoretical_loss": 3.446572722378289, + "tokens_seen": 1890844672 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343636363636364, + "loss": 1.365, + "theoretical_loss": 3.4465626522839745, + "tokens_seen": 1890910208 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.34188660979270935, + "objective/train/avg_token_score": 0.036958470940589905, + "objective/train/docs_used": 1069021, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.070196270942688, + "objective/train/original_loss": 4.393176555633545, + "objective/train/theoretical_loss": 3.4465525826363885, + "objective/train/tokens_used": 249834976, + "objective/train/unlikelihood_loss": 0.0641588643193245, + "theoretical_loss": 3.4465525826363885, + "tokens_seen": 1890975744 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343434343434344, + "loss": 1.2818, + "theoretical_loss": 3.4465525826363885, + "tokens_seen": 1890975744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043432323232323234, + "loss": 1.4453, + "theoretical_loss": 3.4465425134354946, + "tokens_seen": 1891041280 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343030303030303, + "loss": 1.2402, + "theoretical_loss": 3.4465324446812575, + "tokens_seen": 1891106816 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043428282828282826, + "loss": 1.1511, + "theoretical_loss": 3.4465223763736423, + "tokens_seen": 1891172352 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043426262626262627, + "loss": 1.3436, + "theoretical_loss": 3.446512308512614, + "tokens_seen": 1891237888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342424242424243, + "loss": 1.5911, + "theoretical_loss": 3.446502241098137, + "tokens_seen": 1891303424 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043422222222222224, + "loss": 1.6752, + "theoretical_loss": 3.4464921741301757, + "tokens_seen": 1891368960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342020202020202, + "loss": 1.6683, + "theoretical_loss": 3.4464821076086953, + "tokens_seen": 1891434496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341818181818182, + "loss": 1.3651, + "theoretical_loss": 3.44647204153366, + "tokens_seen": 1891500032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043416161616161616, + "loss": 1.2549, + "theoretical_loss": 3.4464619759050352, + "tokens_seen": 1891565568 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043414141414141417, + "loss": 1.3456, + "theoretical_loss": 3.4464519107227853, + "tokens_seen": 1891631104 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043412121212121213, + "loss": 1.3359, + "theoretical_loss": 3.4464418459868753, + "tokens_seen": 1891696640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341010101010101, + "loss": 1.3899, + "theoretical_loss": 3.446431781697269, + "tokens_seen": 1891762176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340808080808081, + "loss": 1.3479, + "theoretical_loss": 3.4464217178539327, + "tokens_seen": 1891827712 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043406060606060605, + "loss": 1.3248, + "theoretical_loss": 3.4464116544568295, + "tokens_seen": 1891893248 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043404040404040406, + "loss": 1.4866, + "theoretical_loss": 3.446401591505925, + "tokens_seen": 1891958784 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434020202020202, + "loss": 1.5089, + "theoretical_loss": 3.4463915290011844, + "tokens_seen": 1892024320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043400000000000003, + "loss": 1.3789, + "theoretical_loss": 3.446381466942572, + "tokens_seen": 1892089856 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433979797979798, + "loss": 1.4955, + "theoretical_loss": 3.4463714053300523, + "tokens_seen": 1892155392 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043395959595959595, + "loss": 1.3215, + "theoretical_loss": 3.44636134416359, + "tokens_seen": 1892220928 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043393939393939396, + "loss": 1.3518, + "theoretical_loss": 3.4463512834431507, + "tokens_seen": 1892286464 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339191919191919, + "loss": 1.3355, + "theoretical_loss": 3.446341223168698, + "tokens_seen": 1892352000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338989898989899, + "loss": 1.2708, + "theoretical_loss": 3.446331163340198, + "tokens_seen": 1892417536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338787878787879, + "loss": 1.2829, + "theoretical_loss": 3.446321103957615, + "tokens_seen": 1892483072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043385858585858584, + "loss": 1.3394, + "theoretical_loss": 3.4463110450209133, + "tokens_seen": 1892548608 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.34042033553123474, + "objective/train/avg_token_score": 0.015821443870663643, + "objective/train/docs_used": 1070317, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1134051084518433, + "objective/train/original_loss": 4.933949947357178, + "objective/train/theoretical_loss": 3.4463009865300585, + "objective/train/tokens_used": 251473376, + "objective/train/unlikelihood_loss": 0.06462860852479935, + "theoretical_loss": 3.4463009865300585, + "tokens_seen": 1892614144 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043383838383838385, + "loss": 1.2695, + "theoretical_loss": 3.4463009865300585, + "tokens_seen": 1892614144 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043381818181818186, + "loss": 1.4255, + "theoretical_loss": 3.4462909284850145, + "tokens_seen": 1892679680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337979797979798, + "loss": 1.2214, + "theoretical_loss": 3.4462808708857464, + "tokens_seen": 1892745216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337777777777778, + "loss": 1.2181, + "theoretical_loss": 3.4462708137322196, + "tokens_seen": 1892810752 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043375757575757573, + "loss": 1.5435, + "theoretical_loss": 3.446260757024398, + "tokens_seen": 1892876288 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043373737373737374, + "loss": 1.3984, + "theoretical_loss": 3.4462507007622474, + "tokens_seen": 1892941824 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043371717171717175, + "loss": 1.3259, + "theoretical_loss": 3.4462406449457315, + "tokens_seen": 1893007360 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336969696969697, + "loss": 1.3695, + "theoretical_loss": 3.4462305895748164, + "tokens_seen": 1893072896 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043367676767676767, + "loss": 1.2128, + "theoretical_loss": 3.4462205346494654, + "tokens_seen": 1893138432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336565656565656, + "loss": 1.6229, + "theoretical_loss": 3.446210480169645, + "tokens_seen": 1893203968 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043363636363636364, + "loss": 1.2829, + "theoretical_loss": 3.446200426135319, + "tokens_seen": 1893269504 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043361616161616165, + "loss": 1.6587, + "theoretical_loss": 3.4461903725464524, + "tokens_seen": 1893335040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335959595959596, + "loss": 1.1699, + "theoretical_loss": 3.4461803194030103, + "tokens_seen": 1893400576 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043357575757575756, + "loss": 1.2126, + "theoretical_loss": 3.446170266704957, + "tokens_seen": 1893466112 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043355555555555557, + "loss": 1.2627, + "theoretical_loss": 3.446160214452258, + "tokens_seen": 1893531648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335353535353536, + "loss": 1.3907, + "theoretical_loss": 3.4461501626448783, + "tokens_seen": 1893597184 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043351515151515154, + "loss": 1.4824, + "theoretical_loss": 3.446140111282782, + "tokens_seen": 1893662720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334949494949495, + "loss": 1.3936, + "theoretical_loss": 3.4461300603659337, + "tokens_seen": 1893728256 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043347474747474745, + "loss": 1.3941, + "theoretical_loss": 3.4461200098942997, + "tokens_seen": 1893793792 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043345454545454546, + "loss": 1.3449, + "theoretical_loss": 3.4461099598678437, + "tokens_seen": 1893859328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334343434343435, + "loss": 1.7087, + "theoretical_loss": 3.446099910286531, + "tokens_seen": 1893924864 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043341414141414143, + "loss": 1.8037, + "theoretical_loss": 3.446089861150326, + "tokens_seen": 1893990400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333939393939394, + "loss": 1.5121, + "theoretical_loss": 3.4460798124591947, + "tokens_seen": 1894055936 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043337373737373735, + "loss": 1.377, + "theoretical_loss": 3.446069764213101, + "tokens_seen": 1894121472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333535353535354, + "loss": 1.3279, + "theoretical_loss": 3.4460597164120097, + "tokens_seen": 1894187008 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.36241447925567627, + "objective/train/avg_token_score": 0.02387678064405918, + "objective/train/docs_used": 1070823, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2387202978134155, + "objective/train/original_loss": 4.809488296508789, + "objective/train/theoretical_loss": 3.4460496690558866, + "objective/train/tokens_used": 253111776, + "objective/train/unlikelihood_loss": 0.04541734606027603, + "theoretical_loss": 3.4460496690558866, + "tokens_seen": 1894252544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043333333333333337, + "loss": 1.2874, + "theoretical_loss": 3.4460496690558866, + "tokens_seen": 1894252544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333131313131313, + "loss": 1.5159, + "theoretical_loss": 3.4460396221446956, + "tokens_seen": 1894318080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332929292929293, + "loss": 1.2886, + "theoretical_loss": 3.4460295756784025, + "tokens_seen": 1894383616 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043327272727272724, + "loss": 1.3834, + "theoretical_loss": 3.446019529656972, + "tokens_seen": 1894449152 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332525252525253, + "loss": 1.3738, + "theoretical_loss": 3.446009484080368, + "tokens_seen": 1894514688 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043323232323232326, + "loss": 1.4346, + "theoretical_loss": 3.445999438948557, + "tokens_seen": 1894580224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332121212121212, + "loss": 1.415, + "theoretical_loss": 3.4459893942615025, + "tokens_seen": 1894645760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331919191919192, + "loss": 1.3809, + "theoretical_loss": 3.4459793500191704, + "tokens_seen": 1894711296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331717171717172, + "loss": 1.5446, + "theoretical_loss": 3.4459693062215253, + "tokens_seen": 1894776832 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331515151515152, + "loss": 1.5414, + "theoretical_loss": 3.445959262868532, + "tokens_seen": 1894842368 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043313131313131315, + "loss": 1.1756, + "theoretical_loss": 3.4459492199601556, + "tokens_seen": 1894907904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331111111111111, + "loss": 1.5137, + "theoretical_loss": 3.445939177496361, + "tokens_seen": 1894973440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043309090909090907, + "loss": 1.2745, + "theoretical_loss": 3.445929135477113, + "tokens_seen": 1895038976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330707070707071, + "loss": 1.6113, + "theoretical_loss": 3.445919093902377, + "tokens_seen": 1895104512 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330505050505051, + "loss": 1.4817, + "theoretical_loss": 3.4459090527721177, + "tokens_seen": 1895170048 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043303030303030305, + "loss": 1.6125, + "theoretical_loss": 3.4458990120863, + "tokens_seen": 1895235584 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433010101010101, + "loss": 1.345, + "theoretical_loss": 3.445888971844888, + "tokens_seen": 1895301120 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432989898989899, + "loss": 1.4279, + "theoretical_loss": 3.4458789320478487, + "tokens_seen": 1895366656 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043296969696969697, + "loss": 1.4733, + "theoretical_loss": 3.4458688926951453, + "tokens_seen": 1895432192 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432949494949495, + "loss": 1.2651, + "theoretical_loss": 3.4458588537867434, + "tokens_seen": 1895497728 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043292929292929294, + "loss": 1.2785, + "theoretical_loss": 3.4458488153226083, + "tokens_seen": 1895563264 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329090909090909, + "loss": 1.3361, + "theoretical_loss": 3.445838777302704, + "tokens_seen": 1895628800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328888888888889, + "loss": 1.4636, + "theoretical_loss": 3.4458287397269967, + "tokens_seen": 1895694336 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043286868686868686, + "loss": 1.257, + "theoretical_loss": 3.4458187025954503, + "tokens_seen": 1895759872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328484848484849, + "loss": 1.486, + "theoretical_loss": 3.4458086659080305, + "tokens_seen": 1895825408 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.3908846378326416, + "objective/train/avg_token_score": 0.010636522434651852, + "objective/train/docs_used": 1071554, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2562838792800903, + "objective/train/original_loss": 4.368358612060547, + "objective/train/theoretical_loss": 3.4457986296647025, + "objective/train/tokens_used": 254750176, + "objective/train/unlikelihood_loss": 0.06785128265619278, + "theoretical_loss": 3.4457986296647025, + "tokens_seen": 1895890944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043282828282828283, + "loss": 1.4474, + "theoretical_loss": 3.4457986296647025, + "tokens_seen": 1895890944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043280808080808084, + "loss": 1.3487, + "theoretical_loss": 3.4457885938654305, + "tokens_seen": 1895956480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327878787878788, + "loss": 1.4548, + "theoretical_loss": 3.44577855851018, + "tokens_seen": 1896022016 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043276767676767676, + "loss": 1.3746, + "theoretical_loss": 3.4457685235989155, + "tokens_seen": 1896087552 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043274747474747477, + "loss": 1.4116, + "theoretical_loss": 3.445758489131603, + "tokens_seen": 1896153088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327272727272727, + "loss": 1.5319, + "theoretical_loss": 3.4457484551082063, + "tokens_seen": 1896218624 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043270707070707073, + "loss": 1.5303, + "theoretical_loss": 3.4457384215286915, + "tokens_seen": 1896284160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326868686868687, + "loss": 1.2872, + "theoretical_loss": 3.4457283883930234, + "tokens_seen": 1896349696 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043266666666666665, + "loss": 1.2291, + "theoretical_loss": 3.4457183557011666, + "tokens_seen": 1896415232 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043264646464646466, + "loss": 1.1889, + "theoretical_loss": 3.4457083234530863, + "tokens_seen": 1896480768 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043262626262626267, + "loss": 1.3665, + "theoretical_loss": 3.445698291648747, + "tokens_seen": 1896546304 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043260606060606063, + "loss": 1.3058, + "theoretical_loss": 3.445688260288115, + "tokens_seen": 1896611840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325858585858586, + "loss": 1.4865, + "theoretical_loss": 3.445678229371154, + "tokens_seen": 1896677376 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043256565656565654, + "loss": 1.4227, + "theoretical_loss": 3.4456681988978306, + "tokens_seen": 1896742912 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043254545454545455, + "loss": 1.0513, + "theoretical_loss": 3.4456581688681087, + "tokens_seen": 1896808448 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043252525252525256, + "loss": 1.4629, + "theoretical_loss": 3.4456481392819533, + "tokens_seen": 1896873984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325050505050505, + "loss": 1.2892, + "theoretical_loss": 3.44563811013933, + "tokens_seen": 1896939520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324848484848485, + "loss": 1.5847, + "theoretical_loss": 3.445628081440203, + "tokens_seen": 1897005056 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043246464646464643, + "loss": 1.2676, + "theoretical_loss": 3.445618053184539, + "tokens_seen": 1897070592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324444444444445, + "loss": 1.4628, + "theoretical_loss": 3.4456080253723016, + "tokens_seen": 1897136128 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043242424242424246, + "loss": 1.5347, + "theoretical_loss": 3.445597998003456, + "tokens_seen": 1897201664 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324040404040404, + "loss": 1.38, + "theoretical_loss": 3.445587971077968, + "tokens_seen": 1897267200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043238383838383837, + "loss": 1.2716, + "theoretical_loss": 3.4455779445958026, + "tokens_seen": 1897332736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323636363636363, + "loss": 1.4612, + "theoretical_loss": 3.4455679185569243, + "tokens_seen": 1897398272 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323434343434344, + "loss": 1.404, + "theoretical_loss": 3.4455578929612987, + "tokens_seen": 1897463808 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.2730327546596527, + "objective/train/avg_token_score": 0.044983237981796265, + "objective/train/docs_used": 1072893, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9510472416877747, + "objective/train/original_loss": 5.534091949462891, + "objective/train/theoretical_loss": 3.44554786780889, + "objective/train/tokens_used": 256388576, + "objective/train/unlikelihood_loss": 0.05126022920012474, + "theoretical_loss": 3.44554786780889, + "tokens_seen": 1897529344 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043232323232323235, + "loss": 1.0224, + "theoretical_loss": 3.44554786780889, + "tokens_seen": 1897529344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323030303030303, + "loss": 1.3145, + "theoretical_loss": 3.4455378430996646, + "tokens_seen": 1897594880 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043228282828282826, + "loss": 1.354, + "theoretical_loss": 3.445527818833587, + "tokens_seen": 1897660416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322626262626262, + "loss": 1.3433, + "theoretical_loss": 3.4455177950106215, + "tokens_seen": 1897725952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322424242424243, + "loss": 1.1633, + "theoretical_loss": 3.445507771630735, + "tokens_seen": 1897791488 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043222222222222224, + "loss": 1.7054, + "theoretical_loss": 3.4454977486938914, + "tokens_seen": 1897857024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322020202020202, + "loss": 1.3594, + "theoretical_loss": 3.4454877262000556, + "tokens_seen": 1897922560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043218181818181816, + "loss": 1.3057, + "theoretical_loss": 3.4454777041491935, + "tokens_seen": 1897988096 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043216161616161617, + "loss": 1.1239, + "theoretical_loss": 3.44546768254127, + "tokens_seen": 1898053632 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321414141414142, + "loss": 1.3014, + "theoretical_loss": 3.44545766137625, + "tokens_seen": 1898119168 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043212121212121213, + "loss": 1.5642, + "theoretical_loss": 3.4454476406540984, + "tokens_seen": 1898184704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321010101010101, + "loss": 1.1932, + "theoretical_loss": 3.445437620374781, + "tokens_seen": 1898250240 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043208080808080805, + "loss": 1.3947, + "theoretical_loss": 3.445427600538263, + "tokens_seen": 1898315776 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043206060606060606, + "loss": 1.2837, + "theoretical_loss": 3.4454175811445085, + "tokens_seen": 1898381312 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043204040404040407, + "loss": 1.4581, + "theoretical_loss": 3.445407562193484, + "tokens_seen": 1898446848 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432020202020202, + "loss": 1.3282, + "theoretical_loss": 3.4453975436851536, + "tokens_seen": 1898512384 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432, + "loss": 1.2793, + "theoretical_loss": 3.445387525619483, + "tokens_seen": 1898577920 + }, + { + "epoch": 0.14, + "learning_rate": 0.000431979797979798, + "loss": 1.1152, + "theoretical_loss": 3.445377507996437, + "tokens_seen": 1898643456 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043195959595959595, + "loss": 1.4725, + "theoretical_loss": 3.445367490815981, + "tokens_seen": 1898708992 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043193939393939396, + "loss": 1.2346, + "theoretical_loss": 3.4453574740780804, + "tokens_seen": 1898774528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319191919191919, + "loss": 1.4218, + "theoretical_loss": 3.4453474577826997, + "tokens_seen": 1898840064 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318989898989899, + "loss": 1.2056, + "theoretical_loss": 3.445337441929805, + "tokens_seen": 1898905600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318787878787879, + "loss": 1.2807, + "theoretical_loss": 3.445327426519361, + "tokens_seen": 1898971136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318585858585859, + "loss": 1.4656, + "theoretical_loss": 3.4453174115513323, + "tokens_seen": 1899036672 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043183838383838386, + "loss": 1.4633, + "theoretical_loss": 3.445307397025685, + "tokens_seen": 1899102208 + }, + { + "epoch": 0.14, + "objective/train/aligned_token_count": 0.42344820499420166, + "objective/train/avg_token_score": 0.01560823805630207, + "objective/train/docs_used": 1073632, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4783105850219727, + "objective/train/original_loss": 4.294405937194824, + "objective/train/theoretical_loss": 3.445297382942384, + "objective/train/tokens_used": 258026976, + "objective/train/unlikelihood_loss": 0.07553756982088089, + "theoretical_loss": 3.445297382942384, + "tokens_seen": 1899167744 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318181818181818, + "loss": 1.2705, + "theoretical_loss": 3.445297382942384, + "tokens_seen": 1899167744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317979797979798, + "loss": 1.622, + "theoretical_loss": 3.4452873693013943, + "tokens_seen": 1899233280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317777777777778, + "loss": 1.4875, + "theoretical_loss": 3.4452773561026815, + "tokens_seen": 1899298816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317575757575758, + "loss": 1.5592, + "theoretical_loss": 3.4452673433462104, + "tokens_seen": 1899364352 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043173737373737375, + "loss": 1.3319, + "theoretical_loss": 3.4452573310319465, + "tokens_seen": 1899429888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317171717171717, + "loss": 1.4507, + "theoretical_loss": 3.4452473191598547, + "tokens_seen": 1899495424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316969696969697, + "loss": 1.4265, + "theoretical_loss": 3.4452373077299003, + "tokens_seen": 1899560960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316767676767677, + "loss": 1.4393, + "theoretical_loss": 3.4452272967420488, + "tokens_seen": 1899626496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316565656565657, + "loss": 1.5585, + "theoretical_loss": 3.4452172861962653, + "tokens_seen": 1899692032 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043163636363636364, + "loss": 1.3088, + "theoretical_loss": 3.445207276092515, + "tokens_seen": 1899757568 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043161616161616165, + "loss": 1.4769, + "theoretical_loss": 3.445197266430763, + "tokens_seen": 1899823104 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315959595959596, + "loss": 1.2856, + "theoretical_loss": 3.4451872572109745, + "tokens_seen": 1899888640 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043157575757575757, + "loss": 1.4165, + "theoretical_loss": 3.445177248433115, + "tokens_seen": 1899954176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315555555555556, + "loss": 1.5391, + "theoretical_loss": 3.44516724009715, + "tokens_seen": 1900019712 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043153535353535353, + "loss": 1.4838, + "theoretical_loss": 3.445157232203044, + "tokens_seen": 1900085248 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043151515151515154, + "loss": 1.2202, + "theoretical_loss": 3.4451472247507624, + "tokens_seen": 1900150784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314949494949495, + "loss": 1.3836, + "theoretical_loss": 3.445137217740271, + "tokens_seen": 1900216320 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043147474747474746, + "loss": 1.0373, + "theoretical_loss": 3.4451272111715348, + "tokens_seen": 1900281856 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043145454545454547, + "loss": 1.2555, + "theoretical_loss": 3.445117205044519, + "tokens_seen": 1900347392 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314343434343435, + "loss": 1.3939, + "theoretical_loss": 3.445107199359189, + "tokens_seen": 1900412928 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043141414141414144, + "loss": 1.4978, + "theoretical_loss": 3.44509719411551, + "tokens_seen": 1900478464 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313939393939394, + "loss": 1.5079, + "theoretical_loss": 3.4450871893134467, + "tokens_seen": 1900544000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043137373737373735, + "loss": 1.2719, + "theoretical_loss": 3.4450771849529653, + "tokens_seen": 1900609536 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043135353535353536, + "loss": 1.3437, + "theoretical_loss": 3.4450671810340308, + "tokens_seen": 1900675072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313333333333334, + "loss": 1.3644, + "theoretical_loss": 3.4450571775566083, + "tokens_seen": 1900740608 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.41379520297050476, + "objective/train/avg_token_score": 0.014362385496497154, + "objective/train/docs_used": 1075027, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3654704093933105, + "objective/train/original_loss": 4.367673397064209, + "objective/train/theoretical_loss": 3.445047174520663, + "objective/train/tokens_used": 259665376, + "objective/train/unlikelihood_loss": 0.07200692594051361, + "theoretical_loss": 3.445047174520663, + "tokens_seen": 1900806144 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043131313131313133, + "loss": 1.5024, + "theoretical_loss": 3.445047174520663, + "tokens_seen": 1900806144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312929292929293, + "loss": 1.1957, + "theoretical_loss": 3.4450371719261605, + "tokens_seen": 1900871680 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043127272727272724, + "loss": 1.3585, + "theoretical_loss": 3.4450271697730663, + "tokens_seen": 1900937216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312525252525253, + "loss": 1.4242, + "theoretical_loss": 3.445017168061345, + "tokens_seen": 1901002752 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043123232323232327, + "loss": 1.5416, + "theoretical_loss": 3.4450071667909623, + "tokens_seen": 1901068288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312121212121212, + "loss": 1.5098, + "theoretical_loss": 3.4449971659618837, + "tokens_seen": 1901133824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311919191919192, + "loss": 1.3824, + "theoretical_loss": 3.4449871655740743, + "tokens_seen": 1901199360 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043117171717171714, + "loss": 1.3807, + "theoretical_loss": 3.444977165627499, + "tokens_seen": 1901264896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311515151515152, + "loss": 1.4715, + "theoretical_loss": 3.444967166122124, + "tokens_seen": 1901330432 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043113131313131316, + "loss": 1.228, + "theoretical_loss": 3.444957167057914, + "tokens_seen": 1901395968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311111111111111, + "loss": 1.4939, + "theoretical_loss": 3.444947168434835, + "tokens_seen": 1901461504 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043109090909090907, + "loss": 1.4963, + "theoretical_loss": 3.4449371702528513, + "tokens_seen": 1901527040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310707070707071, + "loss": 1.336, + "theoretical_loss": 3.444927172511929, + "tokens_seen": 1901592576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310505050505051, + "loss": 1.2116, + "theoretical_loss": 3.4449171752120336, + "tokens_seen": 1901658112 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043103030303030305, + "loss": 1.2582, + "theoretical_loss": 3.4449071783531293, + "tokens_seen": 1901723648 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431010101010101, + "loss": 1.5321, + "theoretical_loss": 3.4448971819351826, + "tokens_seen": 1901789184 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043098989898989897, + "loss": 1.5561, + "theoretical_loss": 3.4448871859581587, + "tokens_seen": 1901854720 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430969696969697, + "loss": 1.478, + "theoretical_loss": 3.444877190422023, + "tokens_seen": 1901920256 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430949494949495, + "loss": 1.5553, + "theoretical_loss": 3.44486719532674, + "tokens_seen": 1901985792 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043092929292929294, + "loss": 1.5061, + "theoretical_loss": 3.444857200672276, + "tokens_seen": 1902051328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309090909090909, + "loss": 1.4788, + "theoretical_loss": 3.4448472064585958, + "tokens_seen": 1902116864 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308888888888889, + "loss": 1.5469, + "theoretical_loss": 3.4448372126856652, + "tokens_seen": 1902182400 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043086868686868687, + "loss": 1.4974, + "theoretical_loss": 3.4448272193534493, + "tokens_seen": 1902247936 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308484848484849, + "loss": 1.3048, + "theoretical_loss": 3.4448172264619137, + "tokens_seen": 1902313472 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043082828282828284, + "loss": 1.5208, + "theoretical_loss": 3.444807234011024, + "tokens_seen": 1902379008 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.4122678339481354, + "objective/train/avg_token_score": 0.011399907991290092, + "objective/train/docs_used": 1075669, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.282902479171753, + "objective/train/original_loss": 4.224909782409668, + "objective/train/theoretical_loss": 3.4447972420007447, + "objective/train/tokens_used": 261303776, + "objective/train/unlikelihood_loss": 0.0683131068944931, + "theoretical_loss": 3.4447972420007447, + "tokens_seen": 1902444544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308080808080808, + "loss": 1.3168, + "theoretical_loss": 3.4447972420007447, + "tokens_seen": 1902444544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307878787878788, + "loss": 1.4082, + "theoretical_loss": 3.444787250431042, + "tokens_seen": 1902510080 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043076767676767676, + "loss": 1.5043, + "theoretical_loss": 3.444777259301881, + "tokens_seen": 1902575616 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043074747474747477, + "loss": 1.6162, + "theoretical_loss": 3.4447672686132274, + "tokens_seen": 1902641152 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043072727272727273, + "loss": 1.3416, + "theoretical_loss": 3.4447572783650457, + "tokens_seen": 1902706688 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043070707070707074, + "loss": 1.2339, + "theoretical_loss": 3.444747288557303, + "tokens_seen": 1902772224 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306868686868687, + "loss": 1.5255, + "theoretical_loss": 3.444737299189963, + "tokens_seen": 1902837760 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043066666666666665, + "loss": 1.3823, + "theoretical_loss": 3.444727310262992, + "tokens_seen": 1902903296 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043064646464646467, + "loss": 1.3543, + "theoretical_loss": 3.444717321776355, + "tokens_seen": 1902968832 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306262626262626, + "loss": 1.4874, + "theoretical_loss": 3.444707333730018, + "tokens_seen": 1903034368 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043060606060606063, + "loss": 1.5816, + "theoretical_loss": 3.444697346123946, + "tokens_seen": 1903099904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305858585858586, + "loss": 1.3016, + "theoretical_loss": 3.444687358958104, + "tokens_seen": 1903165440 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043056565656565655, + "loss": 1.3505, + "theoretical_loss": 3.4446773722324586, + "tokens_seen": 1903230976 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043054545454545456, + "loss": 1.4311, + "theoretical_loss": 3.4446673859469747, + "tokens_seen": 1903296512 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043052525252525257, + "loss": 1.5355, + "theoretical_loss": 3.444657400101617, + "tokens_seen": 1903362048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305050505050505, + "loss": 1.3983, + "theoretical_loss": 3.444647414696352, + "tokens_seen": 1903427584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304848484848485, + "loss": 1.2973, + "theoretical_loss": 3.444637429731145, + "tokens_seen": 1903493120 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043046464646464644, + "loss": 1.5914, + "theoretical_loss": 3.4446274452059606, + "tokens_seen": 1903558656 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043044444444444445, + "loss": 1.3239, + "theoretical_loss": 3.444617461120765, + "tokens_seen": 1903624192 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043042424242424246, + "loss": 1.2012, + "theoretical_loss": 3.444607477475524, + "tokens_seen": 1903689728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304040404040404, + "loss": 1.4485, + "theoretical_loss": 3.4445974942702025, + "tokens_seen": 1903755264 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303838383838384, + "loss": 1.4015, + "theoretical_loss": 3.4445875115047655, + "tokens_seen": 1903820800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043036363636363633, + "loss": 1.5879, + "theoretical_loss": 3.4445775291791794, + "tokens_seen": 1903886336 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043034343434343434, + "loss": 1.4073, + "theoretical_loss": 3.444567547293409, + "tokens_seen": 1903951872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043032323232323235, + "loss": 1.3852, + "theoretical_loss": 3.444557565847421, + "tokens_seen": 1904017408 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.3672409653663635, + "objective/train/avg_token_score": 0.005817563273012638, + "objective/train/docs_used": 1076839, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2039426565170288, + "objective/train/original_loss": 3.8227851390838623, + "objective/train/theoretical_loss": 3.444547584841179, + "objective/train/tokens_used": 262942176, + "objective/train/unlikelihood_loss": 0.13508202135562897, + "theoretical_loss": 3.444547584841179, + "tokens_seen": 1904082944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303030303030303, + "loss": 1.2922, + "theoretical_loss": 3.444547584841179, + "tokens_seen": 1904082944 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043028282828282827, + "loss": 1.2261, + "theoretical_loss": 3.4445376042746503, + "tokens_seen": 1904148480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302626262626262, + "loss": 1.3418, + "theoretical_loss": 3.444527624147799, + "tokens_seen": 1904214016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302424242424243, + "loss": 1.402, + "theoretical_loss": 3.444517644460591, + "tokens_seen": 1904279552 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043022222222222225, + "loss": 1.406, + "theoretical_loss": 3.4445076652129925, + "tokens_seen": 1904345088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302020202020202, + "loss": 1.3695, + "theoretical_loss": 3.4444976864049686, + "tokens_seen": 1904410624 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043018181818181816, + "loss": 1.5536, + "theoretical_loss": 3.444487708036484, + "tokens_seen": 1904476160 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043016161616161617, + "loss": 1.4343, + "theoretical_loss": 3.444477730107505, + "tokens_seen": 1904541696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301414141414142, + "loss": 1.1865, + "theoretical_loss": 3.4444677526179976, + "tokens_seen": 1904607232 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043012121212121214, + "loss": 1.6126, + "theoretical_loss": 3.444457775567927, + "tokens_seen": 1904672768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301010101010101, + "loss": 1.6826, + "theoretical_loss": 3.444447798957258, + "tokens_seen": 1904738304 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043008080808080805, + "loss": 1.335, + "theoretical_loss": 3.4444378227859564, + "tokens_seen": 1904803840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300606060606061, + "loss": 1.4046, + "theoretical_loss": 3.4444278470539884, + "tokens_seen": 1904869376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300404040404041, + "loss": 1.5168, + "theoretical_loss": 3.4444178717613188, + "tokens_seen": 1904934912 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043002020202020203, + "loss": 1.4559, + "theoretical_loss": 3.4444078969079133, + "tokens_seen": 1905000448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043, + "loss": 1.3272, + "theoretical_loss": 3.4443979224937378, + "tokens_seen": 1905065984 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042997979797979795, + "loss": 1.3365, + "theoretical_loss": 3.4443879485187576, + "tokens_seen": 1905131520 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429959595959596, + "loss": 1.4219, + "theoretical_loss": 3.4443779749829386, + "tokens_seen": 1905197056 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042993939393939397, + "loss": 1.5434, + "theoretical_loss": 3.4443680018862457, + "tokens_seen": 1905262592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004299191919191919, + "loss": 1.5114, + "theoretical_loss": 3.4443580292286446, + "tokens_seen": 1905328128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298989898989899, + "loss": 1.5505, + "theoretical_loss": 3.4443480570101017, + "tokens_seen": 1905393664 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298787878787879, + "loss": 1.3701, + "theoretical_loss": 3.444338085230582, + "tokens_seen": 1905459200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298585858585859, + "loss": 1.3049, + "theoretical_loss": 3.44432811389005, + "tokens_seen": 1905524736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042983838383838386, + "loss": 1.4485, + "theoretical_loss": 3.444318142988473, + "tokens_seen": 1905590272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298181818181818, + "loss": 1.4768, + "theoretical_loss": 3.444308172525816, + "tokens_seen": 1905655808 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.35905423760414124, + "objective/train/avg_token_score": 0.026558930054306984, + "objective/train/docs_used": 1077487, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1562389135360718, + "objective/train/original_loss": 4.9117536544799805, + "objective/train/theoretical_loss": 3.444298202502044, + "objective/train/tokens_used": 264580576, + "objective/train/unlikelihood_loss": 0.038373254239559174, + "theoretical_loss": 3.444298202502044, + "tokens_seen": 1905721344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297979797979798, + "loss": 1.3541, + "theoretical_loss": 3.444298202502044, + "tokens_seen": 1905721344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297777777777778, + "loss": 1.3404, + "theoretical_loss": 3.444288232917123, + "tokens_seen": 1905786880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297575757575758, + "loss": 1.3965, + "theoretical_loss": 3.444278263771019, + "tokens_seen": 1905852416 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042973737373737375, + "loss": 1.3603, + "theoretical_loss": 3.444268295063697, + "tokens_seen": 1905917952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297171717171717, + "loss": 1.5391, + "theoretical_loss": 3.444258326795123, + "tokens_seen": 1905983488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296969696969697, + "loss": 1.5438, + "theoretical_loss": 3.444248358965263, + "tokens_seen": 1906049024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296767676767677, + "loss": 1.3518, + "theoretical_loss": 3.444238391574081, + "tokens_seen": 1906114560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296565656565657, + "loss": 1.1733, + "theoretical_loss": 3.4442284246215444, + "tokens_seen": 1906180096 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042963636363636365, + "loss": 1.4349, + "theoretical_loss": 3.444218458107618, + "tokens_seen": 1906245632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296161616161616, + "loss": 1.436, + "theoretical_loss": 3.444208492032267, + "tokens_seen": 1906311168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295959595959596, + "loss": 1.3842, + "theoretical_loss": 3.444198526395458, + "tokens_seen": 1906376704 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042957575757575757, + "loss": 1.3577, + "theoretical_loss": 3.444188561197156, + "tokens_seen": 1906442240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295555555555556, + "loss": 1.4073, + "theoretical_loss": 3.4441785964373266, + "tokens_seen": 1906507776 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042953535353535354, + "loss": 1.3905, + "theoretical_loss": 3.444168632115936, + "tokens_seen": 1906573312 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042951515151515155, + "loss": 1.3146, + "theoretical_loss": 3.444158668232949, + "tokens_seen": 1906638848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004294949494949495, + "loss": 1.4867, + "theoretical_loss": 3.444148704788332, + "tokens_seen": 1906704384 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042947474747474746, + "loss": 1.5472, + "theoretical_loss": 3.44413874178205, + "tokens_seen": 1906769920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004294545454545455, + "loss": 1.4212, + "theoretical_loss": 3.4441287792140693, + "tokens_seen": 1906835456 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042943434343434343, + "loss": 1.2909, + "theoretical_loss": 3.4441188170843553, + "tokens_seen": 1906900992 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042941414141414144, + "loss": 1.4079, + "theoretical_loss": 3.444108855392874, + "tokens_seen": 1906966528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293939393939394, + "loss": 1.4177, + "theoretical_loss": 3.44409889413959, + "tokens_seen": 1907032064 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042937373737373736, + "loss": 1.4536, + "theoretical_loss": 3.4440889333244695, + "tokens_seen": 1907097600 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042935353535353537, + "loss": 1.4919, + "theoretical_loss": 3.444078972947479, + "tokens_seen": 1907163136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293333333333334, + "loss": 1.3301, + "theoretical_loss": 3.4440690130085825, + "tokens_seen": 1907228672 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042931313131313134, + "loss": 1.3173, + "theoretical_loss": 3.4440590535077478, + "tokens_seen": 1907294208 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.3209921717643738, + "objective/train/avg_token_score": 0.06907281279563904, + "objective/train/docs_used": 1078017, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0279090404510498, + "objective/train/original_loss": 4.60770320892334, + "objective/train/theoretical_loss": 3.4440490944449387, + "objective/train/tokens_used": 266218976, + "objective/train/unlikelihood_loss": 0.046941839158535004, + "theoretical_loss": 3.4440490944449387, + "tokens_seen": 1907359744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292929292929293, + "loss": 1.2658, + "theoretical_loss": 3.4440490944449387, + "tokens_seen": 1907359744 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042927272727272725, + "loss": 1.43, + "theoretical_loss": 3.444039135820122, + "tokens_seen": 1907425280 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042925252525252526, + "loss": 1.4694, + "theoretical_loss": 3.444029177633263, + "tokens_seen": 1907490816 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042923232323232327, + "loss": 1.2123, + "theoretical_loss": 3.4440192198843276, + "tokens_seen": 1907556352 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042921212121212123, + "loss": 1.3888, + "theoretical_loss": 3.4440092625732808, + "tokens_seen": 1907621888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291919191919192, + "loss": 1.3047, + "theoretical_loss": 3.443999305700089, + "tokens_seen": 1907687424 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042917171717171714, + "loss": 1.2473, + "theoretical_loss": 3.443989349264718, + "tokens_seen": 1907752960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291515151515152, + "loss": 1.4243, + "theoretical_loss": 3.4439793932671323, + "tokens_seen": 1907818496 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042913131313131316, + "loss": 1.4856, + "theoretical_loss": 3.4439694377073, + "tokens_seen": 1907884032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291111111111111, + "loss": 1.5347, + "theoretical_loss": 3.4439594825851843, + "tokens_seen": 1907949568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290909090909091, + "loss": 1.5656, + "theoretical_loss": 3.4439495279007524, + "tokens_seen": 1908015104 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042907070707070703, + "loss": 1.3704, + "theoretical_loss": 3.443939573653969, + "tokens_seen": 1908080640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290505050505051, + "loss": 1.2791, + "theoretical_loss": 3.443929619844801, + "tokens_seen": 1908146176 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042903030303030306, + "loss": 1.1324, + "theoretical_loss": 3.4439196664732137, + "tokens_seen": 1908211712 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429010101010101, + "loss": 1.5249, + "theoretical_loss": 3.4439097135391723, + "tokens_seen": 1908277248 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042898989898989897, + "loss": 1.5048, + "theoretical_loss": 3.443899761042643, + "tokens_seen": 1908342784 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042896969696969693, + "loss": 1.5334, + "theoretical_loss": 3.4438898089835916, + "tokens_seen": 1908408320 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428949494949495, + "loss": 1.6011, + "theoretical_loss": 3.443879857361984, + "tokens_seen": 1908473856 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042892929292929295, + "loss": 1.4087, + "theoretical_loss": 3.4438699061777855, + "tokens_seen": 1908539392 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289090909090909, + "loss": 1.3535, + "theoretical_loss": 3.4438599554309617, + "tokens_seen": 1908604928 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042888888888888886, + "loss": 1.4036, + "theoretical_loss": 3.4438500051214787, + "tokens_seen": 1908670464 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288686868686869, + "loss": 1.2635, + "theoretical_loss": 3.4438400552493027, + "tokens_seen": 1908736000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288484848484849, + "loss": 1.3974, + "theoretical_loss": 3.4438301058143987, + "tokens_seen": 1908801536 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042882828282828284, + "loss": 1.5568, + "theoretical_loss": 3.443820156816733, + "tokens_seen": 1908867072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288080808080808, + "loss": 1.4838, + "theoretical_loss": 3.443810208256271, + "tokens_seen": 1908932608 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.37292277812957764, + "objective/train/avg_token_score": 0.02143978700041771, + "objective/train/docs_used": 1079511, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.331836462020874, + "objective/train/original_loss": 4.501890659332275, + "objective/train/theoretical_loss": 3.4438002601329787, + "objective/train/tokens_used": 267857376, + "objective/train/unlikelihood_loss": 0.05701581388711929, + "theoretical_loss": 3.4438002601329787, + "tokens_seen": 1908998144 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042878787878787876, + "loss": 1.5142, + "theoretical_loss": 3.4438002601329787, + "tokens_seen": 1908998144 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042876767676767677, + "loss": 1.4552, + "theoretical_loss": 3.443790312446822, + "tokens_seen": 1909063680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287474747474748, + "loss": 1.1977, + "theoretical_loss": 3.4437803651977665, + "tokens_seen": 1909129216 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042872727272727274, + "loss": 1.5264, + "theoretical_loss": 3.4437704183857782, + "tokens_seen": 1909194752 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287070707070707, + "loss": 1.3258, + "theoretical_loss": 3.443760472010822, + "tokens_seen": 1909260288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286868686868687, + "loss": 1.3133, + "theoretical_loss": 3.443750526072865, + "tokens_seen": 1909325824 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042866666666666666, + "loss": 1.2519, + "theoretical_loss": 3.4437405805718724, + "tokens_seen": 1909391360 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042864646464646467, + "loss": 1.282, + "theoretical_loss": 3.4437306355078094, + "tokens_seen": 1909456896 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042862626262626263, + "loss": 1.3893, + "theoretical_loss": 3.4437206908806433, + "tokens_seen": 1909522432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286060606060606, + "loss": 1.2844, + "theoretical_loss": 3.4437107466903383, + "tokens_seen": 1909587968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285858585858586, + "loss": 1.5531, + "theoretical_loss": 3.443700802936861, + "tokens_seen": 1909653504 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042856565656565655, + "loss": 1.5443, + "theoretical_loss": 3.443690859620178, + "tokens_seen": 1909719040 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042854545454545456, + "loss": 1.4003, + "theoretical_loss": 3.443680916740253, + "tokens_seen": 1909784576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285252525252525, + "loss": 1.4069, + "theoretical_loss": 3.443670974297054, + "tokens_seen": 1909850112 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042850505050505053, + "loss": 1.574, + "theoretical_loss": 3.443661032290546, + "tokens_seen": 1909915648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284848484848485, + "loss": 1.5845, + "theoretical_loss": 3.4436510907206945, + "tokens_seen": 1909981184 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042846464646464645, + "loss": 1.2667, + "theoretical_loss": 3.4436411495874655, + "tokens_seen": 1910046720 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042844444444444446, + "loss": 1.1777, + "theoretical_loss": 3.4436312088908254, + "tokens_seen": 1910112256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284242424242424, + "loss": 1.3442, + "theoretical_loss": 3.4436212686307393, + "tokens_seen": 1910177792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284040404040404, + "loss": 1.3153, + "theoretical_loss": 3.4436113288071732, + "tokens_seen": 1910243328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283838383838384, + "loss": 1.5457, + "theoretical_loss": 3.4436013894200936, + "tokens_seen": 1910308864 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283636363636364, + "loss": 1.2672, + "theoretical_loss": 3.443591450469466, + "tokens_seen": 1910374400 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042834343434343435, + "loss": 1.084, + "theoretical_loss": 3.4435815119552555, + "tokens_seen": 1910439936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042832323232323236, + "loss": 1.3617, + "theoretical_loss": 3.443571573877429, + "tokens_seen": 1910505472 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283030303030303, + "loss": 1.3787, + "theoretical_loss": 3.443561636235952, + "tokens_seen": 1910571008 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.5610948204994202, + "objective/train/avg_token_score": 0.003200286766514182, + "objective/train/docs_used": 1080108, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.690970540046692, + "objective/train/original_loss": 3.2980706691741943, + "objective/train/theoretical_loss": 3.4435516990307904, + "objective/train/tokens_used": 269495776, + "objective/train/unlikelihood_loss": 0.08375748991966248, + "theoretical_loss": 3.4435516990307904, + "tokens_seen": 1910636544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282828282828283, + "loss": 1.4718, + "theoretical_loss": 3.4435516990307904, + "tokens_seen": 1910636544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282626262626263, + "loss": 1.5774, + "theoretical_loss": 3.44354176226191, + "tokens_seen": 1910702080 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042824242424242424, + "loss": 1.6015, + "theoretical_loss": 3.4435318259292766, + "tokens_seen": 1910767616 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042822222222222225, + "loss": 1.3461, + "theoretical_loss": 3.443521890032856, + "tokens_seen": 1910833152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282020202020202, + "loss": 1.394, + "theoretical_loss": 3.4435119545726147, + "tokens_seen": 1910898688 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042818181818181817, + "loss": 1.3769, + "theoretical_loss": 3.443502019548518, + "tokens_seen": 1910964224 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281616161616162, + "loss": 1.1004, + "theoretical_loss": 3.4434920849605324, + "tokens_seen": 1911029760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281414141414142, + "loss": 1.5169, + "theoretical_loss": 3.443482150808623, + "tokens_seen": 1911095296 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042812121212121215, + "loss": 1.4129, + "theoretical_loss": 3.443472217092756, + "tokens_seen": 1911160832 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281010101010101, + "loss": 1.3754, + "theoretical_loss": 3.443462283812898, + "tokens_seen": 1911226368 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042808080808080806, + "loss": 1.3636, + "theoretical_loss": 3.4434523509690136, + "tokens_seen": 1911291904 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042806060606060607, + "loss": 1.2775, + "theoretical_loss": 3.44344241856107, + "tokens_seen": 1911357440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280404040404041, + "loss": 1.6295, + "theoretical_loss": 3.4434324865890322, + "tokens_seen": 1911422976 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042802020202020204, + "loss": 1.5199, + "theoretical_loss": 3.4434225550528668, + "tokens_seen": 1911488512 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428, + "loss": 1.3486, + "theoretical_loss": 3.443412623952539, + "tokens_seen": 1911554048 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042797979797979795, + "loss": 1.4287, + "theoretical_loss": 3.4434026932880153, + "tokens_seen": 1911619584 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427959595959596, + "loss": 1.4952, + "theoretical_loss": 3.4433927630592613, + "tokens_seen": 1911685120 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427939393939394, + "loss": 1.6055, + "theoretical_loss": 3.4433828332662437, + "tokens_seen": 1911750656 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042791919191919193, + "loss": 1.3703, + "theoretical_loss": 3.4433729039089274, + "tokens_seen": 1911816192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278989898989899, + "loss": 1.3665, + "theoretical_loss": 3.443362974987279, + "tokens_seen": 1911881728 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042787878787878784, + "loss": 1.3683, + "theoretical_loss": 3.4433530465012643, + "tokens_seen": 1911947264 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278585858585859, + "loss": 1.3692, + "theoretical_loss": 3.443343118450849, + "tokens_seen": 1912012800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042783838383838387, + "loss": 1.4151, + "theoretical_loss": 3.4433331908359994, + "tokens_seen": 1912078336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278181818181818, + "loss": 1.4771, + "theoretical_loss": 3.4433232636566817, + "tokens_seen": 1912143872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277979797979798, + "loss": 1.4327, + "theoretical_loss": 3.443313336912861, + "tokens_seen": 1912209408 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.33864858746528625, + "objective/train/avg_token_score": 0.007881486788392067, + "objective/train/docs_used": 1081319, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1766334772109985, + "objective/train/original_loss": 4.45033597946167, + "objective/train/theoretical_loss": 3.443303410604504, + "objective/train/tokens_used": 271134176, + "objective/train/unlikelihood_loss": 0.05650121346116066, + "theoretical_loss": 3.443303410604504, + "tokens_seen": 1912274944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277777777777778, + "loss": 1.4087, + "theoretical_loss": 3.443303410604504, + "tokens_seen": 1912274944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277575757575758, + "loss": 1.6281, + "theoretical_loss": 3.4432934847315764, + "tokens_seen": 1912340480 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042773737373737376, + "loss": 1.4231, + "theoretical_loss": 3.443283559294044, + "tokens_seen": 1912406016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277171717171717, + "loss": 1.3221, + "theoretical_loss": 3.4432736342918737, + "tokens_seen": 1912471552 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276969696969697, + "loss": 1.3538, + "theoretical_loss": 3.44326370972503, + "tokens_seen": 1912537088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276767676767677, + "loss": 1.3519, + "theoretical_loss": 3.4432537855934804, + "tokens_seen": 1912602624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276565656565657, + "loss": 1.2999, + "theoretical_loss": 3.44324386189719, + "tokens_seen": 1912668160 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042763636363636365, + "loss": 1.4608, + "theoretical_loss": 3.4432339386361246, + "tokens_seen": 1912733696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276161616161616, + "loss": 1.5103, + "theoretical_loss": 3.4432240158102507, + "tokens_seen": 1912799232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275959595959596, + "loss": 1.3832, + "theoretical_loss": 3.443214093419534, + "tokens_seen": 1912864768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275757575757576, + "loss": 1.319, + "theoretical_loss": 3.4432041714639414, + "tokens_seen": 1912930304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275555555555556, + "loss": 1.2985, + "theoretical_loss": 3.4431942499434376, + "tokens_seen": 1912995840 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042753535353535354, + "loss": 1.2791, + "theoretical_loss": 3.4431843288579893, + "tokens_seen": 1913061376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275151515151515, + "loss": 1.3884, + "theoretical_loss": 3.443174408207563, + "tokens_seen": 1913126912 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274949494949495, + "loss": 1.3111, + "theoretical_loss": 3.4431644879921235, + "tokens_seen": 1913192448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042747474747474747, + "loss": 1.4369, + "theoretical_loss": 3.4431545682116376, + "tokens_seen": 1913257984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274545454545455, + "loss": 1.2989, + "theoretical_loss": 3.4431446488660713, + "tokens_seen": 1913323520 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042743434343434344, + "loss": 1.5759, + "theoretical_loss": 3.44313472995539, + "tokens_seen": 1913389056 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042741414141414145, + "loss": 1.5069, + "theoretical_loss": 3.443124811479561, + "tokens_seen": 1913454592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273939393939394, + "loss": 1.3957, + "theoretical_loss": 3.44311489343855, + "tokens_seen": 1913520128 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042737373737373736, + "loss": 1.4563, + "theoretical_loss": 3.443104975832322, + "tokens_seen": 1913585664 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273535353535354, + "loss": 1.1546, + "theoretical_loss": 3.4430950586608438, + "tokens_seen": 1913651200 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042733333333333333, + "loss": 1.3923, + "theoretical_loss": 3.4430851419240818, + "tokens_seen": 1913716736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042731313131313134, + "loss": 1.3045, + "theoretical_loss": 3.4430752256220014, + "tokens_seen": 1913782272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272929292929293, + "loss": 1.2177, + "theoretical_loss": 3.443065309754569, + "tokens_seen": 1913847808 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.29844820499420166, + "objective/train/avg_token_score": 0.011274569667875767, + "objective/train/docs_used": 1081978, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.975776731967926, + "objective/train/original_loss": 4.575207710266113, + "objective/train/theoretical_loss": 3.4430553943217506, + "objective/train/tokens_used": 272772576, + "objective/train/unlikelihood_loss": 0.051495835185050964, + "theoretical_loss": 3.4430553943217506, + "tokens_seen": 1913913344 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042727272727272726, + "loss": 1.4363, + "theoretical_loss": 3.4430553943217506, + "tokens_seen": 1913913344 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042725252525252527, + "loss": 1.283, + "theoretical_loss": 3.443045479323512, + "tokens_seen": 1913978880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272323232323233, + "loss": 1.5076, + "theoretical_loss": 3.4430355647598194, + "tokens_seen": 1914044416 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042721212121212123, + "loss": 1.2358, + "theoretical_loss": 3.4430256506306396, + "tokens_seen": 1914109952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271919191919192, + "loss": 1.4152, + "theoretical_loss": 3.443015736935938, + "tokens_seen": 1914175488 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042717171717171715, + "loss": 1.2006, + "theoretical_loss": 3.4430058236756804, + "tokens_seen": 1914241024 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042715151515151516, + "loss": 1.5529, + "theoretical_loss": 3.4429959108498336, + "tokens_seen": 1914306560 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042713131313131317, + "loss": 1.3738, + "theoretical_loss": 3.4429859984583633, + "tokens_seen": 1914372096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271111111111111, + "loss": 1.4871, + "theoretical_loss": 3.4429760865012353, + "tokens_seen": 1914437632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270909090909091, + "loss": 1.3976, + "theoretical_loss": 3.442966174978416, + "tokens_seen": 1914503168 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042707070707070704, + "loss": 1.1469, + "theoretical_loss": 3.4429562638898723, + "tokens_seen": 1914568704 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042705050505050505, + "loss": 1.2692, + "theoretical_loss": 3.442946353235569, + "tokens_seen": 1914634240 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042703030303030306, + "loss": 1.5126, + "theoretical_loss": 3.4429364430154727, + "tokens_seen": 1914699776 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427010101010101, + "loss": 1.4688, + "theoretical_loss": 3.44292653322955, + "tokens_seen": 1914765312 + }, + { + "epoch": 0.15, + "learning_rate": 0.000426989898989899, + "loss": 1.4204, + "theoretical_loss": 3.4429166238777658, + "tokens_seen": 1914830848 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042696969696969693, + "loss": 1.5229, + "theoretical_loss": 3.442906714960088, + "tokens_seen": 1914896384 + }, + { + "epoch": 0.15, + "learning_rate": 0.000426949494949495, + "loss": 1.504, + "theoretical_loss": 3.4428968064764813, + "tokens_seen": 1914961920 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042692929292929296, + "loss": 1.3443, + "theoretical_loss": 3.442886898426912, + "tokens_seen": 1915027456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269090909090909, + "loss": 1.2391, + "theoretical_loss": 3.442876990811347, + "tokens_seen": 1915092992 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042688888888888887, + "loss": 1.1644, + "theoretical_loss": 3.4428670836297517, + "tokens_seen": 1915158528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268686868686868, + "loss": 1.4366, + "theoretical_loss": 3.442857176882093, + "tokens_seen": 1915224064 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268484848484849, + "loss": 1.6832, + "theoretical_loss": 3.442847270568336, + "tokens_seen": 1915289600 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042682828282828285, + "loss": 1.41, + "theoretical_loss": 3.4428373646884474, + "tokens_seen": 1915355136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268080808080808, + "loss": 1.37, + "theoretical_loss": 3.4428274592423938, + "tokens_seen": 1915420672 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042678787878787876, + "loss": 1.3526, + "theoretical_loss": 3.4428175542301407, + "tokens_seen": 1915486208 + }, + { + "epoch": 0.15, + "objective/train/aligned_token_count": 0.41758307814598083, + "objective/train/avg_token_score": 0.019459545612335205, + "objective/train/docs_used": 1083212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3157575130462646, + "objective/train/original_loss": 4.584113121032715, + "objective/train/theoretical_loss": 3.442807649651654, + "objective/train/tokens_used": 274410976, + "objective/train/unlikelihood_loss": 0.04653491452336311, + "theoretical_loss": 3.442807649651654, + "tokens_seen": 1915551744 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042676767676767677, + "loss": 1.4367, + "theoretical_loss": 3.442807649651654, + "tokens_seen": 1915551744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004267474747474748, + "loss": 1.189, + "theoretical_loss": 3.4427977455069008, + "tokens_seen": 1915617280 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042672727272727274, + "loss": 1.3401, + "theoretical_loss": 3.4427878417958464, + "tokens_seen": 1915682816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004267070707070707, + "loss": 1.2258, + "theoretical_loss": 3.442777938518458, + "tokens_seen": 1915748352 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042668686868686865, + "loss": 1.505, + "theoretical_loss": 3.4427680356747006, + "tokens_seen": 1915813888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266666666666667, + "loss": 1.6968, + "theoretical_loss": 3.442758133264541, + "tokens_seen": 1915879424 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266464646464647, + "loss": 1.431, + "theoretical_loss": 3.442748231287946, + "tokens_seen": 1915944960 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042662626262626263, + "loss": 1.4045, + "theoretical_loss": 3.4427383297448806, + "tokens_seen": 1916010496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266060606060606, + "loss": 1.2996, + "theoretical_loss": 3.4427284286353115, + "tokens_seen": 1916076032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265858585858586, + "loss": 1.403, + "theoretical_loss": 3.442718527959205, + "tokens_seen": 1916141568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265656565656566, + "loss": 1.3892, + "theoretical_loss": 3.4427086277165273, + "tokens_seen": 1916207104 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042654545454545457, + "loss": 1.5321, + "theoretical_loss": 3.442698727907244, + "tokens_seen": 1916272640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265252525252525, + "loss": 1.5272, + "theoretical_loss": 3.442688828531322, + "tokens_seen": 1916338176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265050505050505, + "loss": 1.5984, + "theoretical_loss": 3.4426789295887277, + "tokens_seen": 1916403712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264848484848485, + "loss": 1.3496, + "theoretical_loss": 3.4426690310794266, + "tokens_seen": 1916469248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264646464646465, + "loss": 1.3557, + "theoretical_loss": 3.442659133003385, + "tokens_seen": 1916534784 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042644444444444446, + "loss": 1.2853, + "theoretical_loss": 3.4426492353605695, + "tokens_seen": 1916600320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264242424242424, + "loss": 1.5258, + "theoretical_loss": 3.442639338150946, + "tokens_seen": 1916665856 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042640404040404043, + "loss": 1.2848, + "theoretical_loss": 3.4426294413744816, + "tokens_seen": 1916731392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263838383838384, + "loss": 1.4453, + "theoretical_loss": 3.4426195450311416, + "tokens_seen": 1916796928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263636363636364, + "loss": 1.2794, + "theoretical_loss": 3.442609649120892, + "tokens_seen": 1916862464 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042634343434343435, + "loss": 1.2195, + "theoretical_loss": 3.4425997536437, + "tokens_seen": 1916928000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263232323232323, + "loss": 1.6612, + "theoretical_loss": 3.4425898585995314, + "tokens_seen": 1916993536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263030303030303, + "loss": 1.3784, + "theoretical_loss": 3.4425799639883525, + "tokens_seen": 1917059072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262828282828283, + "loss": 1.2362, + "theoretical_loss": 3.442570069810129, + "tokens_seen": 1917124608 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.3271627426147461, + "objective/train/avg_token_score": 0.00935707613825798, + "objective/train/docs_used": 1083753, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1364368200302124, + "objective/train/original_loss": 4.213170051574707, + "objective/train/theoretical_loss": 3.442560176064828, + "objective/train/tokens_used": 276049376, + "objective/train/unlikelihood_loss": 0.05666195973753929, + "theoretical_loss": 3.442560176064828, + "tokens_seen": 1917190144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262626262626263, + "loss": 1.2487, + "theoretical_loss": 3.442560176064828, + "tokens_seen": 1917190144 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042624242424242425, + "loss": 1.3266, + "theoretical_loss": 3.442550282752415, + "tokens_seen": 1917255680 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042622222222222226, + "loss": 1.4237, + "theoretical_loss": 3.4425403898728573, + "tokens_seen": 1917321216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262020202020202, + "loss": 1.3311, + "theoretical_loss": 3.44253049742612, + "tokens_seen": 1917386752 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042618181818181817, + "loss": 1.4461, + "theoretical_loss": 3.44252060541217, + "tokens_seen": 1917452288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261616161616162, + "loss": 1.5083, + "theoretical_loss": 3.4425107138309734, + "tokens_seen": 1917517824 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042614141414141414, + "loss": 1.5493, + "theoretical_loss": 3.4425008226824967, + "tokens_seen": 1917583360 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042612121212121215, + "loss": 1.3401, + "theoretical_loss": 3.442490931966706, + "tokens_seen": 1917648896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261010101010101, + "loss": 1.521, + "theoretical_loss": 3.4424810416835676, + "tokens_seen": 1917714432 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042608080808080806, + "loss": 1.3547, + "theoretical_loss": 3.4424711518330477, + "tokens_seen": 1917779968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260606060606061, + "loss": 1.4561, + "theoretical_loss": 3.442461262415113, + "tokens_seen": 1917845504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260404040404041, + "loss": 1.4914, + "theoretical_loss": 3.442451373429729, + "tokens_seen": 1917911040 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042602020202020204, + "loss": 1.2794, + "theoretical_loss": 3.4424414848768627, + "tokens_seen": 1917976576 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426, + "loss": 1.2327, + "theoretical_loss": 3.4424315967564807, + "tokens_seen": 1918042112 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042597979797979796, + "loss": 1.5526, + "theoretical_loss": 3.4424217090685483, + "tokens_seen": 1918107648 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042595959595959597, + "loss": 1.5492, + "theoretical_loss": 3.4424118218130326, + "tokens_seen": 1918173184 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425939393939394, + "loss": 1.3568, + "theoretical_loss": 3.442401934989899, + "tokens_seen": 1918238720 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042591919191919194, + "loss": 1.3991, + "theoretical_loss": 3.442392048599115, + "tokens_seen": 1918304256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258989898989899, + "loss": 1.3583, + "theoretical_loss": 3.442382162640646, + "tokens_seen": 1918369792 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042587878787878785, + "loss": 1.5928, + "theoretical_loss": 3.442372277114459, + "tokens_seen": 1918435328 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258585858585859, + "loss": 1.1758, + "theoretical_loss": 3.44236239202052, + "tokens_seen": 1918500864 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042583838383838387, + "loss": 1.4569, + "theoretical_loss": 3.4423525073587955, + "tokens_seen": 1918566400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042581818181818183, + "loss": 1.4608, + "theoretical_loss": 3.4423426231292518, + "tokens_seen": 1918631936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257979797979798, + "loss": 1.5675, + "theoretical_loss": 3.442332739331855, + "tokens_seen": 1918697472 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042577777777777774, + "loss": 1.2314, + "theoretical_loss": 3.4423228559665713, + "tokens_seen": 1918763008 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.4659701883792877, + "objective/train/avg_token_score": 0.018756110221147537, + "objective/train/docs_used": 1084888, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6133241653442383, + "objective/train/original_loss": 4.159404277801514, + "objective/train/theoretical_loss": 3.442312973033368, + "objective/train/tokens_used": 277687776, + "objective/train/unlikelihood_loss": 0.05935472249984741, + "theoretical_loss": 3.442312973033368, + "tokens_seen": 1918828544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257575757575758, + "loss": 1.3151, + "theoretical_loss": 3.442312973033368, + "tokens_seen": 1918828544 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042573737373737377, + "loss": 1.4888, + "theoretical_loss": 3.4423030905322105, + "tokens_seen": 1918894080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257171717171717, + "loss": 1.4874, + "theoretical_loss": 3.4422932084630653, + "tokens_seen": 1918959616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256969696969697, + "loss": 1.4814, + "theoretical_loss": 3.442283326825899, + "tokens_seen": 1919025152 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042567676767676764, + "loss": 1.3381, + "theoretical_loss": 3.4422734456206783, + "tokens_seen": 1919090688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256565656565657, + "loss": 1.4287, + "theoretical_loss": 3.4422635648473685, + "tokens_seen": 1919156224 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042563636363636366, + "loss": 1.2571, + "theoretical_loss": 3.442253684505937, + "tokens_seen": 1919221760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256161616161616, + "loss": 1.3749, + "theoretical_loss": 3.4422438045963495, + "tokens_seen": 1919287296 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042559595959595957, + "loss": 1.6466, + "theoretical_loss": 3.4422339251185727, + "tokens_seen": 1919352832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255757575757576, + "loss": 1.3934, + "theoretical_loss": 3.4422240460725737, + "tokens_seen": 1919418368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255555555555556, + "loss": 1.2283, + "theoretical_loss": 3.4422141674583173, + "tokens_seen": 1919483904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042553535353535355, + "loss": 1.3776, + "theoretical_loss": 3.4422042892757716, + "tokens_seen": 1919549440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255151515151515, + "loss": 1.4192, + "theoretical_loss": 3.4421944115249015, + "tokens_seen": 1919614976 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042549494949494946, + "loss": 1.4077, + "theoretical_loss": 3.442184534205674, + "tokens_seen": 1919680512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254747474747475, + "loss": 1.3611, + "theoretical_loss": 3.4421746573180556, + "tokens_seen": 1919746048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254545454545455, + "loss": 1.3919, + "theoretical_loss": 3.4421647808620133, + "tokens_seen": 1919811584 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042543434343434344, + "loss": 1.6059, + "theoretical_loss": 3.442154904837512, + "tokens_seen": 1919877120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254141414141414, + "loss": 1.4137, + "theoretical_loss": 3.4421450292445197, + "tokens_seen": 1919942656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253939393939394, + "loss": 1.4409, + "theoretical_loss": 3.442135154083002, + "tokens_seen": 1920008192 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042537373737373737, + "loss": 1.2995, + "theoretical_loss": 3.4421252793529247, + "tokens_seen": 1920073728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253535353535354, + "loss": 1.457, + "theoretical_loss": 3.4421154050542553, + "tokens_seen": 1920139264 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042533333333333334, + "loss": 1.3346, + "theoretical_loss": 3.4421055311869604, + "tokens_seen": 1920204800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253131313131313, + "loss": 1.4856, + "theoretical_loss": 3.442095657751006, + "tokens_seen": 1920270336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252929292929293, + "loss": 1.4931, + "theoretical_loss": 3.4420857847463573, + "tokens_seen": 1920335872 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042527272727272726, + "loss": 1.4235, + "theoretical_loss": 3.442075912172983, + "tokens_seen": 1920401408 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.2597140669822693, + "objective/train/avg_token_score": 0.00819999910891056, + "objective/train/docs_used": 1085471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8639290928840637, + "objective/train/original_loss": 3.9054853916168213, + "objective/train/theoretical_loss": 3.4420660400308476, + "objective/train/tokens_used": 279326176, + "objective/train/unlikelihood_loss": 0.08530934154987335, + "theoretical_loss": 3.4420660400308476, + "tokens_seen": 1920466944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042525252525252527, + "loss": 1.29, + "theoretical_loss": 3.4420660400308476, + "tokens_seen": 1920466944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042523232323232323, + "loss": 1.4392, + "theoretical_loss": 3.442056168319919, + "tokens_seen": 1920532480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042521212121212124, + "loss": 1.5414, + "theoretical_loss": 3.442046297040163, + "tokens_seen": 1920598016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251919191919192, + "loss": 1.5928, + "theoretical_loss": 3.442036426191546, + "tokens_seen": 1920663552 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042517171717171715, + "loss": 1.5551, + "theoretical_loss": 3.442026555774034, + "tokens_seen": 1920729088 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042515151515151516, + "loss": 1.4526, + "theoretical_loss": 3.442016685787595, + "tokens_seen": 1920794624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251313131313131, + "loss": 1.3303, + "theoretical_loss": 3.442006816232194, + "tokens_seen": 1920860160 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042511111111111113, + "loss": 1.4716, + "theoretical_loss": 3.441996947107798, + "tokens_seen": 1920925696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250909090909091, + "loss": 1.7319, + "theoretical_loss": 3.441987078414373, + "tokens_seen": 1920991232 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042507070707070705, + "loss": 1.298, + "theoretical_loss": 3.4419772101518866, + "tokens_seen": 1921056768 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042505050505050506, + "loss": 1.4499, + "theoretical_loss": 3.4419673423203037, + "tokens_seen": 1921122304 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042503030303030307, + "loss": 1.4579, + "theoretical_loss": 3.4419574749195925, + "tokens_seen": 1921187840 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425010101010101, + "loss": 1.3127, + "theoretical_loss": 3.441947607949718, + "tokens_seen": 1921253376 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424989898989899, + "loss": 1.3889, + "theoretical_loss": 3.441937741410648, + "tokens_seen": 1921318912 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424969696969697, + "loss": 1.3067, + "theoretical_loss": 3.441927875302348, + "tokens_seen": 1921384448 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042494949494949495, + "loss": 1.2578, + "theoretical_loss": 3.441918009624785, + "tokens_seen": 1921449984 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042492929292929296, + "loss": 1.2618, + "theoretical_loss": 3.4419081443779254, + "tokens_seen": 1921515520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249090909090909, + "loss": 1.5736, + "theoretical_loss": 3.4418982795617357, + "tokens_seen": 1921581056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248888888888889, + "loss": 1.4578, + "theoretical_loss": 3.441888415176182, + "tokens_seen": 1921646592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248686868686869, + "loss": 1.3573, + "theoretical_loss": 3.4418785512212313, + "tokens_seen": 1921712128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248484848484849, + "loss": 1.3088, + "theoretical_loss": 3.44186868769685, + "tokens_seen": 1921777664 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042482828282828285, + "loss": 1.2699, + "theoretical_loss": 3.441858824603005, + "tokens_seen": 1921843200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248080808080808, + "loss": 1.4885, + "theoretical_loss": 3.4418489619396624, + "tokens_seen": 1921908736 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042478787878787877, + "loss": 1.4237, + "theoretical_loss": 3.4418390997067885, + "tokens_seen": 1921974272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247676767676768, + "loss": 1.4151, + "theoretical_loss": 3.44182923790435, + "tokens_seen": 1922039808 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.37664955854415894, + "objective/train/avg_token_score": 0.027382804080843925, + "objective/train/docs_used": 1086847, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2897065877914429, + "objective/train/original_loss": 4.506113529205322, + "objective/train/theoretical_loss": 3.4418193765323135, + "objective/train/tokens_used": 280964576, + "objective/train/unlikelihood_loss": 0.06149740517139435, + "theoretical_loss": 3.4418193765323135, + "tokens_seen": 1922105344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247474747474748, + "loss": 1.3919, + "theoretical_loss": 3.4418193765323135, + "tokens_seen": 1922105344 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042472727272727275, + "loss": 1.1067, + "theoretical_loss": 3.4418095155906463, + "tokens_seen": 1922170880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247070707070707, + "loss": 1.3229, + "theoretical_loss": 3.441799655079314, + "tokens_seen": 1922236416 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042468686868686866, + "loss": 1.283, + "theoretical_loss": 3.441789794998283, + "tokens_seen": 1922301952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246666666666667, + "loss": 1.3661, + "theoretical_loss": 3.4417799353475202, + "tokens_seen": 1922367488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246464646464647, + "loss": 1.3491, + "theoretical_loss": 3.4417700761269927, + "tokens_seen": 1922433024 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042462626262626264, + "loss": 1.4425, + "theoretical_loss": 3.4417602173366664, + "tokens_seen": 1922498560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246060606060606, + "loss": 1.5416, + "theoretical_loss": 3.441750358976508, + "tokens_seen": 1922564096 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042458585858585855, + "loss": 1.2915, + "theoretical_loss": 3.4417405010464837, + "tokens_seen": 1922629632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245656565656566, + "loss": 1.7483, + "theoretical_loss": 3.441730643546561, + "tokens_seen": 1922695168 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245454545454546, + "loss": 1.4832, + "theoretical_loss": 3.441720786476706, + "tokens_seen": 1922760704 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042452525252525253, + "loss": 1.5824, + "theoretical_loss": 3.441710929836885, + "tokens_seen": 1922826240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245050505050505, + "loss": 1.5308, + "theoretical_loss": 3.4417010736270646, + "tokens_seen": 1922891776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244848484848485, + "loss": 1.2997, + "theoretical_loss": 3.441691217847212, + "tokens_seen": 1922957312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244646464646465, + "loss": 1.4648, + "theoretical_loss": 3.4416813624972935, + "tokens_seen": 1923022848 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042444444444444447, + "loss": 1.1912, + "theoretical_loss": 3.441671507577275, + "tokens_seen": 1923088384 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244242424242424, + "loss": 1.3946, + "theoretical_loss": 3.441661653087124, + "tokens_seen": 1923153920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244040404040404, + "loss": 1.4577, + "theoretical_loss": 3.4416517990268067, + "tokens_seen": 1923219456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243838383838384, + "loss": 1.2794, + "theoretical_loss": 3.44164194539629, + "tokens_seen": 1923284992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243636363636364, + "loss": 1.4351, + "theoretical_loss": 3.4416320921955403, + "tokens_seen": 1923350528 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042434343434343436, + "loss": 1.4165, + "theoretical_loss": 3.4416222394245235, + "tokens_seen": 1923416064 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243232323232323, + "loss": 1.4822, + "theoretical_loss": 3.441612387083208, + "tokens_seen": 1923481600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042430303030303033, + "loss": 1.2961, + "theoretical_loss": 3.4416025351715587, + "tokens_seen": 1923547136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242828282828283, + "loss": 1.5846, + "theoretical_loss": 3.441592683689543, + "tokens_seen": 1923612672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242626262626263, + "loss": 1.5942, + "theoretical_loss": 3.4415828326371276, + "tokens_seen": 1923678208 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.353128045797348, + "objective/train/avg_token_score": 0.019363651052117348, + "objective/train/docs_used": 1087440, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.210463047027588, + "objective/train/original_loss": 4.447400093078613, + "objective/train/theoretical_loss": 3.4415729820142786, + "objective/train/tokens_used": 282602976, + "objective/train/unlikelihood_loss": 0.08732765913009644, + "theoretical_loss": 3.4415729820142786, + "tokens_seen": 1923743744 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042424242424242425, + "loss": 1.3893, + "theoretical_loss": 3.4415729820142786, + "tokens_seen": 1923743744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242222222222222, + "loss": 1.3323, + "theoretical_loss": 3.441563131820963, + "tokens_seen": 1923809280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242020202020202, + "loss": 1.3105, + "theoretical_loss": 3.4415532820571473, + "tokens_seen": 1923874816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241818181818182, + "loss": 1.3959, + "theoretical_loss": 3.4415434327227983, + "tokens_seen": 1923940352 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241616161616162, + "loss": 1.5941, + "theoretical_loss": 3.441533583817883, + "tokens_seen": 1924005888 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042414141414141415, + "loss": 1.3544, + "theoretical_loss": 3.4415237353423676, + "tokens_seen": 1924071424 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042412121212121216, + "loss": 1.3194, + "theoretical_loss": 3.4415138872962183, + "tokens_seen": 1924136960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241010101010101, + "loss": 1.4216, + "theoretical_loss": 3.4415040396794025, + "tokens_seen": 1924202496 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042408080808080807, + "loss": 1.4374, + "theoretical_loss": 3.441494192491887, + "tokens_seen": 1924268032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240606060606061, + "loss": 1.6304, + "theoretical_loss": 3.4414843457336373, + "tokens_seen": 1924333568 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042404040404040404, + "loss": 1.2735, + "theoretical_loss": 3.441474499404621, + "tokens_seen": 1924399104 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042402020202020205, + "loss": 1.325, + "theoretical_loss": 3.441464653504805, + "tokens_seen": 1924464640 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424, + "loss": 1.6185, + "theoretical_loss": 3.4414548080341554, + "tokens_seen": 1924530176 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042397979797979796, + "loss": 1.3112, + "theoretical_loss": 3.441444962992639, + "tokens_seen": 1924595712 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423959595959596, + "loss": 1.6121, + "theoretical_loss": 3.441435118380223, + "tokens_seen": 1924661248 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042393939393939393, + "loss": 1.4117, + "theoretical_loss": 3.4414252741968734, + "tokens_seen": 1924726784 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042391919191919194, + "loss": 1.3539, + "theoretical_loss": 3.4414154304425573, + "tokens_seen": 1924792320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238989898989899, + "loss": 1.2487, + "theoretical_loss": 3.4414055871172406, + "tokens_seen": 1924857856 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042387878787878786, + "loss": 1.4242, + "theoretical_loss": 3.441395744220891, + "tokens_seen": 1924923392 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042385858585858587, + "loss": 1.4151, + "theoretical_loss": 3.441385901753475, + "tokens_seen": 1924988928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238383838383839, + "loss": 1.4439, + "theoretical_loss": 3.441376059714959, + "tokens_seen": 1925054464 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042381818181818183, + "loss": 1.5184, + "theoretical_loss": 3.44136621810531, + "tokens_seen": 1925120000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237979797979798, + "loss": 1.5182, + "theoretical_loss": 3.4413563769244946, + "tokens_seen": 1925185536 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042377777777777775, + "loss": 1.4028, + "theoretical_loss": 3.441346536172479, + "tokens_seen": 1925251072 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042375757575757576, + "loss": 1.3353, + "theoretical_loss": 3.441336695849231, + "tokens_seen": 1925316608 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.48949170112609863, + "objective/train/avg_token_score": 0.014318176545202732, + "objective/train/docs_used": 1088918, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6369242668151855, + "objective/train/original_loss": 4.034110069274902, + "objective/train/theoretical_loss": 3.441326855954716, + "objective/train/tokens_used": 284241376, + "objective/train/unlikelihood_loss": 0.05818219855427742, + "theoretical_loss": 3.441326855954716, + "tokens_seen": 1925382144 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042373737373737377, + "loss": 1.5529, + "theoretical_loss": 3.441326855954716, + "tokens_seen": 1925382144 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042371717171717173, + "loss": 1.3521, + "theoretical_loss": 3.4413170164889024, + "tokens_seen": 1925447680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236969696969697, + "loss": 1.267, + "theoretical_loss": 3.441307177451755, + "tokens_seen": 1925513216 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042367676767676764, + "loss": 1.6063, + "theoretical_loss": 3.4412973388432424, + "tokens_seen": 1925578752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236565656565657, + "loss": 1.4146, + "theoretical_loss": 3.44128750066333, + "tokens_seen": 1925644288 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042363636363636366, + "loss": 1.6882, + "theoretical_loss": 3.4412776629119852, + "tokens_seen": 1925709824 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236161616161616, + "loss": 1.3858, + "theoretical_loss": 3.441267825589174, + "tokens_seen": 1925775360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004235959595959596, + "loss": 1.4042, + "theoretical_loss": 3.4412579886948644, + "tokens_seen": 1925840896 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042357575757575753, + "loss": 1.3035, + "theoretical_loss": 3.441248152229022, + "tokens_seen": 1925906432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004235555555555556, + "loss": 1.4533, + "theoretical_loss": 3.4412383161916145, + "tokens_seen": 1925971968 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042353535353535356, + "loss": 1.743, + "theoretical_loss": 3.4412284805826077, + "tokens_seen": 1926037504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004235151515151515, + "loss": 1.2599, + "theoretical_loss": 3.441218645401969, + "tokens_seen": 1926103040 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042349494949494947, + "loss": 1.4499, + "theoretical_loss": 3.441208810649665, + "tokens_seen": 1926168576 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234747474747475, + "loss": 1.5833, + "theoretical_loss": 3.4411989763256625, + "tokens_seen": 1926234112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234545454545455, + "loss": 1.3539, + "theoretical_loss": 3.441189142429928, + "tokens_seen": 1926299648 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042343434343434345, + "loss": 1.4448, + "theoretical_loss": 3.441179308962429, + "tokens_seen": 1926365184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234141414141414, + "loss": 1.485, + "theoretical_loss": 3.4411694759231315, + "tokens_seen": 1926430720 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042339393939393936, + "loss": 1.6433, + "theoretical_loss": 3.4411596433120026, + "tokens_seen": 1926496256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233737373737374, + "loss": 1.311, + "theoretical_loss": 3.4411498111290095, + "tokens_seen": 1926561792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233535353535354, + "loss": 1.408, + "theoretical_loss": 3.4411399793741184, + "tokens_seen": 1926627328 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042333333333333334, + "loss": 1.3597, + "theoretical_loss": 3.441130148047296, + "tokens_seen": 1926692864 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233131313131313, + "loss": 1.4085, + "theoretical_loss": 3.44112031714851, + "tokens_seen": 1926758400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232929292929293, + "loss": 1.2997, + "theoretical_loss": 3.441110486677726, + "tokens_seen": 1926823936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232727272727273, + "loss": 1.3375, + "theoretical_loss": 3.4411006566349114, + "tokens_seen": 1926889472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232525252525253, + "loss": 1.3776, + "theoretical_loss": 3.441090827020033, + "tokens_seen": 1926955008 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.4183773100376129, + "objective/train/avg_token_score": 0.027611570432782173, + "objective/train/docs_used": 1089550, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.40208101272583, + "objective/train/original_loss": 3.778869867324829, + "objective/train/theoretical_loss": 3.4410809978330583, + "objective/train/tokens_used": 285879776, + "objective/train/unlikelihood_loss": 0.07978764176368713, + "theoretical_loss": 3.4410809978330583, + "tokens_seen": 1927020544 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042323232323232323, + "loss": 1.1572, + "theoretical_loss": 3.4410809978330583, + "tokens_seen": 1927020544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232121212121212, + "loss": 1.3984, + "theoretical_loss": 3.441071169073953, + "tokens_seen": 1927086080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231919191919192, + "loss": 1.5348, + "theoretical_loss": 3.441061340742684, + "tokens_seen": 1927151616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231717171717172, + "loss": 1.6712, + "theoretical_loss": 3.4410515128392194, + "tokens_seen": 1927217152 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042315151515151517, + "loss": 1.43, + "theoretical_loss": 3.4410416853635244, + "tokens_seen": 1927282688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231313131313131, + "loss": 1.5115, + "theoretical_loss": 3.441031858315567, + "tokens_seen": 1927348224 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042311111111111114, + "loss": 1.5486, + "theoretical_loss": 3.4410220316953133, + "tokens_seen": 1927413760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230909090909091, + "loss": 1.408, + "theoretical_loss": 3.441012205502731, + "tokens_seen": 1927479296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230707070707071, + "loss": 1.483, + "theoretical_loss": 3.441002379737786, + "tokens_seen": 1927544832 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042305050505050506, + "loss": 1.5122, + "theoretical_loss": 3.4409925544004456, + "tokens_seen": 1927610368 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423030303030303, + "loss": 1.4116, + "theoretical_loss": 3.4409827294906767, + "tokens_seen": 1927675904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042301010101010103, + "loss": 1.561, + "theoretical_loss": 3.4409729050084463, + "tokens_seen": 1927741440 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422989898989899, + "loss": 1.5038, + "theoretical_loss": 3.440963080953721, + "tokens_seen": 1927806976 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422969696969697, + "loss": 1.3527, + "theoretical_loss": 3.440953257326467, + "tokens_seen": 1927872512 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042294949494949496, + "loss": 1.3933, + "theoretical_loss": 3.4409434341266527, + "tokens_seen": 1927938048 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042292929292929297, + "loss": 1.6347, + "theoretical_loss": 3.440933611354244, + "tokens_seen": 1928003584 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229090909090909, + "loss": 1.3893, + "theoretical_loss": 3.4409237890092075, + "tokens_seen": 1928069120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228888888888889, + "loss": 1.0593, + "theoretical_loss": 3.440913967091511, + "tokens_seen": 1928134656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228686868686869, + "loss": 1.2617, + "theoretical_loss": 3.4409041456011202, + "tokens_seen": 1928200192 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042284848484848485, + "loss": 1.2376, + "theoretical_loss": 3.4408943245380033, + "tokens_seen": 1928265728 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042282828282828286, + "loss": 1.4189, + "theoretical_loss": 3.4408845039021263, + "tokens_seen": 1928331264 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228080808080808, + "loss": 1.3413, + "theoretical_loss": 3.4408746836934565, + "tokens_seen": 1928396800 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042278787878787877, + "loss": 1.3963, + "theoretical_loss": 3.4408648639119606, + "tokens_seen": 1928462336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227676767676768, + "loss": 1.2697, + "theoretical_loss": 3.4408550445576056, + "tokens_seen": 1928527872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227474747474748, + "loss": 1.7067, + "theoretical_loss": 3.4408452256303583, + "tokens_seen": 1928593408 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.30810117721557617, + "objective/train/avg_token_score": 0.016539035364985466, + "objective/train/docs_used": 1091004, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9405573606491089, + "objective/train/original_loss": 4.469072341918945, + "objective/train/theoretical_loss": 3.4408354071301854, + "objective/train/tokens_used": 287518176, + "objective/train/unlikelihood_loss": 0.0698821097612381, + "theoretical_loss": 3.4408354071301854, + "tokens_seen": 1928658944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042272727272727275, + "loss": 1.2846, + "theoretical_loss": 3.4408354071301854, + "tokens_seen": 1928658944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227070707070707, + "loss": 1.4077, + "theoretical_loss": 3.4408255890570545, + "tokens_seen": 1928724480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042268686868686867, + "loss": 1.5451, + "theoretical_loss": 3.440815771410932, + "tokens_seen": 1928790016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226666666666667, + "loss": 1.4673, + "theoretical_loss": 3.440805954191785, + "tokens_seen": 1928855552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226464646464647, + "loss": 1.5765, + "theoretical_loss": 3.4407961373995803, + "tokens_seen": 1928921088 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042262626262626264, + "loss": 1.4946, + "theoretical_loss": 3.4407863210342846, + "tokens_seen": 1928986624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226060606060606, + "loss": 1.6541, + "theoretical_loss": 3.440776505095865, + "tokens_seen": 1929052160 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042258585858585856, + "loss": 1.4803, + "theoretical_loss": 3.440766689584289, + "tokens_seen": 1929117696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225656565656566, + "loss": 1.5839, + "theoretical_loss": 3.4407568744995234, + "tokens_seen": 1929183232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225454545454546, + "loss": 1.4868, + "theoretical_loss": 3.4407470598415344, + "tokens_seen": 1929248768 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042252525252525254, + "loss": 1.3395, + "theoretical_loss": 3.440737245610289, + "tokens_seen": 1929314304 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225050505050505, + "loss": 1.437, + "theoretical_loss": 3.4407274318057555, + "tokens_seen": 1929379840 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042248484848484845, + "loss": 1.5503, + "theoretical_loss": 3.440717618427899, + "tokens_seen": 1929445376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224646464646465, + "loss": 1.3331, + "theoretical_loss": 3.4407078054766878, + "tokens_seen": 1929510912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224444444444445, + "loss": 1.4377, + "theoretical_loss": 3.440697992952088, + "tokens_seen": 1929576448 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042242424242424243, + "loss": 1.5041, + "theoretical_loss": 3.4406881808540675, + "tokens_seen": 1929641984 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224040404040404, + "loss": 1.4972, + "theoretical_loss": 3.440678369182592, + "tokens_seen": 1929707520 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042238383838383834, + "loss": 1.4901, + "theoretical_loss": 3.44066855793763, + "tokens_seen": 1929773056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223636363636364, + "loss": 1.3971, + "theoretical_loss": 3.440658747119147, + "tokens_seen": 1929838592 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042234343434343437, + "loss": 1.6506, + "theoretical_loss": 3.440648936727111, + "tokens_seen": 1929904128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223232323232323, + "loss": 1.6115, + "theoretical_loss": 3.4406391267614884, + "tokens_seen": 1929969664 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223030303030303, + "loss": 1.4711, + "theoretical_loss": 3.4406293172222466, + "tokens_seen": 1930035200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222828282828283, + "loss": 1.5098, + "theoretical_loss": 3.4406195081093527, + "tokens_seen": 1930100736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222626262626263, + "loss": 1.3378, + "theoretical_loss": 3.4406096994227733, + "tokens_seen": 1930166272 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042224242424242426, + "loss": 1.5783, + "theoretical_loss": 3.440599891162475, + "tokens_seen": 1930231808 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.6237170100212097, + "objective/train/avg_token_score": 0.006665772292762995, + "objective/train/docs_used": 1091522, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.7739847898483276, + "objective/train/original_loss": 3.4937384128570557, + "objective/train/theoretical_loss": 3.4405900833284258, + "objective/train/tokens_used": 289156576, + "objective/train/unlikelihood_loss": 0.047466766089200974, + "theoretical_loss": 3.4405900833284258, + "tokens_seen": 1930297344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222222222222222, + "loss": 1.4705, + "theoretical_loss": 3.4405900833284258, + "tokens_seen": 1930297344 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042220202020202017, + "loss": 1.3696, + "theoretical_loss": 3.440580275920592, + "tokens_seen": 1930362880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221818181818182, + "loss": 1.3453, + "theoretical_loss": 3.440570468938941, + "tokens_seen": 1930428416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221616161616162, + "loss": 1.6253, + "theoretical_loss": 3.4405606623834393, + "tokens_seen": 1930493952 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042214141414141415, + "loss": 1.5188, + "theoretical_loss": 3.4405508562540543, + "tokens_seen": 1930559488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221212121212121, + "loss": 1.4635, + "theoretical_loss": 3.4405410505507534, + "tokens_seen": 1930625024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221010101010101, + "loss": 1.3429, + "theoretical_loss": 3.440531245273503, + "tokens_seen": 1930690560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220808080808081, + "loss": 1.319, + "theoretical_loss": 3.4405214404222706, + "tokens_seen": 1930756096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220606060606061, + "loss": 1.2327, + "theoretical_loss": 3.4405116359970225, + "tokens_seen": 1930821632 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042204040404040404, + "loss": 1.5396, + "theoretical_loss": 3.4405018319977265, + "tokens_seen": 1930887168 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422020202020202, + "loss": 1.5807, + "theoretical_loss": 3.440492028424349, + "tokens_seen": 1930952704 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422, + "loss": 1.3657, + "theoretical_loss": 3.440482225276858, + "tokens_seen": 1931018240 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042197979797979797, + "loss": 1.2815, + "theoretical_loss": 3.440472422555219, + "tokens_seen": 1931083776 + }, + { + "epoch": 0.16, + "learning_rate": 0.000421959595959596, + "loss": 1.2827, + "theoretical_loss": 3.4404626202594004, + "tokens_seen": 1931149312 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042193939393939394, + "loss": 1.4397, + "theoretical_loss": 3.4404528183893692, + "tokens_seen": 1931214848 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042191919191919195, + "loss": 1.7767, + "theoretical_loss": 3.4404430169450917, + "tokens_seen": 1931280384 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218989898989899, + "loss": 1.6945, + "theoretical_loss": 3.4404332159265354, + "tokens_seen": 1931345920 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042187878787878786, + "loss": 1.3724, + "theoretical_loss": 3.440423415333667, + "tokens_seen": 1931411456 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042185858585858587, + "loss": 1.2764, + "theoretical_loss": 3.440413615166454, + "tokens_seen": 1931476992 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042183838383838383, + "loss": 1.4126, + "theoretical_loss": 3.440403815424864, + "tokens_seen": 1931542528 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042181818181818184, + "loss": 1.3462, + "theoretical_loss": 3.440394016108863, + "tokens_seen": 1931608064 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217979797979798, + "loss": 1.3443, + "theoretical_loss": 3.4403842172184183, + "tokens_seen": 1931673600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042177777777777775, + "loss": 1.5187, + "theoretical_loss": 3.440374418753497, + "tokens_seen": 1931739136 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042175757575757577, + "loss": 1.4158, + "theoretical_loss": 3.4403646207140666, + "tokens_seen": 1931804672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217373737373738, + "loss": 1.5499, + "theoretical_loss": 3.440354823100094, + "tokens_seen": 1931870208 + }, + { + "epoch": 0.16, + "objective/train/aligned_token_count": 0.4935239553451538, + "objective/train/avg_token_score": 0.006382493767887354, + "objective/train/docs_used": 1092471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.711350440979004, + "objective/train/original_loss": 3.7253687381744385, + "objective/train/theoretical_loss": 3.4403450259115465, + "objective/train/tokens_used": 290794976, + "objective/train/unlikelihood_loss": 0.1075059249997139, + "theoretical_loss": 3.4403450259115465, + "tokens_seen": 1931935744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042171717171717173, + "loss": 1.5274, + "theoretical_loss": 3.4403450259115465, + "tokens_seen": 1931935744 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216969696969697, + "loss": 1.6173, + "theoretical_loss": 3.4403352291483906, + "tokens_seen": 1932001280 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042167676767676765, + "loss": 1.55, + "theoretical_loss": 3.440325432810594, + "tokens_seen": 1932066816 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042165656565656566, + "loss": 1.4703, + "theoretical_loss": 3.4403156368981236, + "tokens_seen": 1932132352 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042163636363636367, + "loss": 1.3505, + "theoretical_loss": 3.4403058414109458, + "tokens_seen": 1932197888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216161616161616, + "loss": 1.3519, + "theoretical_loss": 3.4402960463490286, + "tokens_seen": 1932263424 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215959595959596, + "loss": 1.5252, + "theoretical_loss": 3.4402862517123394, + "tokens_seen": 1932328960 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042157575757575754, + "loss": 1.5055, + "theoretical_loss": 3.440276457500844, + "tokens_seen": 1932394496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215555555555556, + "loss": 1.6192, + "theoretical_loss": 3.440266663714511, + "tokens_seen": 1932460032 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042153535353535356, + "loss": 1.56, + "theoretical_loss": 3.440256870353307, + "tokens_seen": 1932525568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215151515151515, + "loss": 1.5159, + "theoretical_loss": 3.4402470774171987, + "tokens_seen": 1932591104 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214949494949495, + "loss": 1.418, + "theoretical_loss": 3.4402372849061535, + "tokens_seen": 1932656640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214747474747475, + "loss": 1.3176, + "theoretical_loss": 3.4402274928201386, + "tokens_seen": 1932722176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214545454545455, + "loss": 1.3761, + "theoretical_loss": 3.4402177011591206, + "tokens_seen": 1932787712 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042143434343434345, + "loss": 1.3156, + "theoretical_loss": 3.4402079099230676, + "tokens_seen": 1932853248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214141414141414, + "loss": 1.4477, + "theoretical_loss": 3.4401981191119466, + "tokens_seen": 1932918784 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042139393939393937, + "loss": 1.3894, + "theoretical_loss": 3.440188328725724, + "tokens_seen": 1932984320 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042137373737373743, + "loss": 1.4207, + "theoretical_loss": 3.4401785387643677, + "tokens_seen": 1933049856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213535353535354, + "loss": 1.3732, + "theoretical_loss": 3.440168749227844, + "tokens_seen": 1933115392 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042133333333333335, + "loss": 1.4545, + "theoretical_loss": 3.440158960116121, + "tokens_seen": 1933180928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213131313131313, + "loss": 1.4779, + "theoretical_loss": 3.4401491714291654, + "tokens_seen": 1933246464 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042129292929292926, + "loss": 1.3513, + "theoretical_loss": 3.4401393831669447, + "tokens_seen": 1933312000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212727272727273, + "loss": 1.4636, + "theoretical_loss": 3.440129595329425, + "tokens_seen": 1933377536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212525252525253, + "loss": 1.3884, + "theoretical_loss": 3.440119807916575, + "tokens_seen": 1933443072 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042123232323232324, + "loss": 1.3359, + "theoretical_loss": 3.440110020928361, + "tokens_seen": 1933508608 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.2790200412273407, + "objective/train/avg_token_score": 0.020743899047374725, + "objective/train/docs_used": 1092955, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.856033444404602, + "objective/train/original_loss": 4.6658196449279785, + "objective/train/theoretical_loss": 3.440100234364751, + "objective/train/tokens_used": 292433376, + "objective/train/unlikelihood_loss": 0.06594061106443405, + "theoretical_loss": 3.440100234364751, + "tokens_seen": 1933574144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212121212121212, + "loss": 1.2959, + "theoretical_loss": 3.440100234364751, + "tokens_seen": 1933574144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211919191919192, + "loss": 1.3046, + "theoretical_loss": 3.4400904482257104, + "tokens_seen": 1933639680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211717171717172, + "loss": 1.4219, + "theoretical_loss": 3.4400806625112086, + "tokens_seen": 1933705216 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211515151515152, + "loss": 1.3372, + "theoretical_loss": 3.440070877221211, + "tokens_seen": 1933770752 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042113131313131313, + "loss": 1.5524, + "theoretical_loss": 3.440061092355686, + "tokens_seen": 1933836288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211111111111111, + "loss": 1.4611, + "theoretical_loss": 3.4400513079146, + "tokens_seen": 1933901824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210909090909091, + "loss": 1.4109, + "theoretical_loss": 3.440041523897921, + "tokens_seen": 1933967360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210707070707071, + "loss": 1.488, + "theoretical_loss": 3.4400317403056153, + "tokens_seen": 1934032896 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042105050505050507, + "loss": 1.3147, + "theoretical_loss": 3.4400219571376507, + "tokens_seen": 1934098432 + }, + { + "epoch": 0.17, + "learning_rate": 0.000421030303030303, + "loss": 1.5583, + "theoretical_loss": 3.4400121743939946, + "tokens_seen": 1934163968 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042101010101010104, + "loss": 1.6352, + "theoretical_loss": 3.4400023920746134, + "tokens_seen": 1934229504 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420989898989899, + "loss": 1.4111, + "theoretical_loss": 3.439992610179475, + "tokens_seen": 1934295040 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420969696969697, + "loss": 1.7752, + "theoretical_loss": 3.4399828287085468, + "tokens_seen": 1934360576 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042094949494949496, + "loss": 1.4621, + "theoretical_loss": 3.4399730476617956, + "tokens_seen": 1934426112 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209292929292929, + "loss": 1.3782, + "theoretical_loss": 3.4399632670391886, + "tokens_seen": 1934491648 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042090909090909093, + "loss": 1.3861, + "theoretical_loss": 3.439953486840693, + "tokens_seen": 1934557184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208888888888889, + "loss": 1.51, + "theoretical_loss": 3.4399437070662766, + "tokens_seen": 1934622720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208686868686869, + "loss": 1.2562, + "theoretical_loss": 3.439933927715906, + "tokens_seen": 1934688256 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042084848484848485, + "loss": 1.3318, + "theoretical_loss": 3.439924148789549, + "tokens_seen": 1934753792 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042082828282828286, + "loss": 1.4035, + "theoretical_loss": 3.4399143702871724, + "tokens_seen": 1934819328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208080808080808, + "loss": 1.5151, + "theoretical_loss": 3.4399045922087437, + "tokens_seen": 1934884864 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207878787878788, + "loss": 1.4422, + "theoretical_loss": 3.4398948145542296, + "tokens_seen": 1934950400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207676767676768, + "loss": 1.4189, + "theoretical_loss": 3.4398850373235987, + "tokens_seen": 1935015936 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042074747474747475, + "loss": 1.4982, + "theoretical_loss": 3.439875260516817, + "tokens_seen": 1935081472 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042072727272727276, + "loss": 1.2191, + "theoretical_loss": 3.439865484133852, + "tokens_seen": 1935147008 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.492118775844574, + "objective/train/avg_token_score": 0.013941711746156216, + "objective/train/docs_used": 1093966, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5089529752731323, + "objective/train/original_loss": 4.055489540100098, + "objective/train/theoretical_loss": 3.4398557081746715, + "objective/train/tokens_used": 294071776, + "objective/train/unlikelihood_loss": 0.046310193836688995, + "theoretical_loss": 3.4398557081746715, + "tokens_seen": 1935212544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207070707070707, + "loss": 1.4559, + "theoretical_loss": 3.4398557081746715, + "tokens_seen": 1935212544 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042068686868686867, + "loss": 1.6231, + "theoretical_loss": 3.439845932639242, + "tokens_seen": 1935278080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004206666666666667, + "loss": 1.3089, + "theoretical_loss": 3.4398361575275316, + "tokens_seen": 1935343616 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042064646464646464, + "loss": 1.6297, + "theoretical_loss": 3.439826382839507, + "tokens_seen": 1935409152 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042062626262626265, + "loss": 1.5352, + "theoretical_loss": 3.4398166085751356, + "tokens_seen": 1935474688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004206060606060606, + "loss": 1.6264, + "theoretical_loss": 3.4398068347343855, + "tokens_seen": 1935540224 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042058585858585856, + "loss": 1.6027, + "theoretical_loss": 3.439797061317223, + "tokens_seen": 1935605760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205656565656566, + "loss": 1.5475, + "theoretical_loss": 3.439787288323615, + "tokens_seen": 1935671296 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205454545454546, + "loss": 1.3246, + "theoretical_loss": 3.43977751575353, + "tokens_seen": 1935736832 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042052525252525254, + "loss": 1.4094, + "theoretical_loss": 3.439767743606935, + "tokens_seen": 1935802368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205050505050505, + "loss": 1.2892, + "theoretical_loss": 3.439757971883797, + "tokens_seen": 1935867904 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042048484848484846, + "loss": 1.4243, + "theoretical_loss": 3.4397482005840834, + "tokens_seen": 1935933440 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042046464646464647, + "loss": 1.6329, + "theoretical_loss": 3.439738429707762, + "tokens_seen": 1935998976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204444444444445, + "loss": 1.4389, + "theoretical_loss": 3.439728659254799, + "tokens_seen": 1936064512 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042042424242424244, + "loss": 1.3894, + "theoretical_loss": 3.439718889225163, + "tokens_seen": 1936130048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204040404040404, + "loss": 1.3502, + "theoretical_loss": 3.4397091196188203, + "tokens_seen": 1936195584 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042038383838383835, + "loss": 1.6292, + "theoretical_loss": 3.4396993504357387, + "tokens_seen": 1936261120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004203636363636364, + "loss": 1.271, + "theoretical_loss": 3.4396895816758857, + "tokens_seen": 1936326656 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042034343434343437, + "loss": 1.6302, + "theoretical_loss": 3.439679813339229, + "tokens_seen": 1936392192 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042032323232323233, + "loss": 1.3143, + "theoretical_loss": 3.439670045425734, + "tokens_seen": 1936457728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004203030303030303, + "loss": 1.597, + "theoretical_loss": 3.4396602779353707, + "tokens_seen": 1936523264 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042028282828282824, + "loss": 1.5688, + "theoretical_loss": 3.4396505108681046, + "tokens_seen": 1936588800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202626262626263, + "loss": 1.5527, + "theoretical_loss": 3.4396407442239036, + "tokens_seen": 1936654336 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042024242424242426, + "loss": 1.5146, + "theoretical_loss": 3.4396309780027354, + "tokens_seen": 1936719872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202222222222222, + "loss": 1.5646, + "theoretical_loss": 3.4396212122045666, + "tokens_seen": 1936785408 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.5265762209892273, + "objective/train/avg_token_score": 0.011445840820670128, + "objective/train/docs_used": 1094637, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9047353267669678, + "objective/train/original_loss": 3.971524477005005, + "objective/train/theoretical_loss": 3.4396114468293657, + "objective/train/tokens_used": 295710176, + "objective/train/unlikelihood_loss": 0.05321503430604935, + "theoretical_loss": 3.4396114468293657, + "tokens_seen": 1936850944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202020202020202, + "loss": 1.4187, + "theoretical_loss": 3.4396114468293657, + "tokens_seen": 1936850944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201818181818182, + "loss": 1.3045, + "theoretical_loss": 3.4396016818770994, + "tokens_seen": 1936916480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201616161616162, + "loss": 1.5315, + "theoretical_loss": 3.4395919173477347, + "tokens_seen": 1936982016 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042014141414141416, + "loss": 1.3532, + "theoretical_loss": 3.43958215324124, + "tokens_seen": 1937047552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201212121212121, + "loss": 1.4987, + "theoretical_loss": 3.439572389557581, + "tokens_seen": 1937113088 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042010101010101007, + "loss": 1.5557, + "theoretical_loss": 3.439562626296727, + "tokens_seen": 1937178624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200808080808081, + "loss": 1.4468, + "theoretical_loss": 3.439552863458644, + "tokens_seen": 1937244160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200606060606061, + "loss": 1.6046, + "theoretical_loss": 3.4395431010433, + "tokens_seen": 1937309696 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042004040404040405, + "loss": 1.3819, + "theoretical_loss": 3.439533339050662, + "tokens_seen": 1937375232 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420020202020202, + "loss": 1.4179, + "theoretical_loss": 3.4395235774806983, + "tokens_seen": 1937440768 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042, + "loss": 1.4089, + "theoretical_loss": 3.439513816333376, + "tokens_seen": 1937506304 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419979797979798, + "loss": 1.379, + "theoretical_loss": 3.439504055608661, + "tokens_seen": 1937571840 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419959595959596, + "loss": 1.4083, + "theoretical_loss": 3.439494295306523, + "tokens_seen": 1937637376 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041993939393939394, + "loss": 1.2468, + "theoretical_loss": 3.439484535426928, + "tokens_seen": 1937702912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199191919191919, + "loss": 1.345, + "theoretical_loss": 3.4394747759698436, + "tokens_seen": 1937768448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198989898989899, + "loss": 1.2737, + "theoretical_loss": 3.439465016935238, + "tokens_seen": 1937833984 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041987878787878787, + "loss": 1.4496, + "theoretical_loss": 3.4394552583230773, + "tokens_seen": 1937899520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198585858585859, + "loss": 1.3009, + "theoretical_loss": 3.4394455001333295, + "tokens_seen": 1937965056 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041983838383838383, + "loss": 1.6033, + "theoretical_loss": 3.439435742365963, + "tokens_seen": 1938030592 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041981818181818185, + "loss": 1.4372, + "theoretical_loss": 3.439425985020944, + "tokens_seen": 1938096128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004197979797979798, + "loss": 1.5184, + "theoretical_loss": 3.43941622809824, + "tokens_seen": 1938161664 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004197777777777778, + "loss": 1.6132, + "theoretical_loss": 3.439406471597819, + "tokens_seen": 1938227200 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041975757575757577, + "loss": 1.4743, + "theoretical_loss": 3.4393967155196483, + "tokens_seen": 1938292736 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041973737373737373, + "loss": 1.5898, + "theoretical_loss": 3.439386959863695, + "tokens_seen": 1938358272 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041971717171717174, + "loss": 1.5563, + "theoretical_loss": 3.439377204629927, + "tokens_seen": 1938423808 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.22458454966545105, + "objective/train/avg_token_score": 0.036178138107061386, + "objective/train/docs_used": 1095318, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.6959583759307861, + "objective/train/original_loss": 4.687398910522461, + "objective/train/theoretical_loss": 3.4393674498183118, + "objective/train/tokens_used": 297348576, + "objective/train/unlikelihood_loss": 0.08731012046337128, + "theoretical_loss": 3.4393674498183118, + "tokens_seen": 1938489344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196969696969697, + "loss": 1.2169, + "theoretical_loss": 3.4393674498183118, + "tokens_seen": 1938489344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196767676767677, + "loss": 1.2217, + "theoretical_loss": 3.439357695428816, + "tokens_seen": 1938554880 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041965656565656566, + "loss": 1.437, + "theoretical_loss": 3.4393479414614085, + "tokens_seen": 1938620416 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196363636363637, + "loss": 1.5371, + "theoretical_loss": 3.4393381879160554, + "tokens_seen": 1938685952 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041961616161616163, + "loss": 1.417, + "theoretical_loss": 3.439328434792725, + "tokens_seen": 1938751488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195959595959596, + "loss": 1.4628, + "theoretical_loss": 3.439318682091385, + "tokens_seen": 1938817024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195757575757576, + "loss": 1.4443, + "theoretical_loss": 3.4393089298120016, + "tokens_seen": 1938882560 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041955555555555556, + "loss": 1.3587, + "theoretical_loss": 3.4392991779545437, + "tokens_seen": 1938948096 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041953535353535357, + "loss": 1.497, + "theoretical_loss": 3.439289426518978, + "tokens_seen": 1939013632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195151515151515, + "loss": 1.6095, + "theoretical_loss": 3.4392796755052712, + "tokens_seen": 1939079168 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194949494949495, + "loss": 1.4658, + "theoretical_loss": 3.439269924913393, + "tokens_seen": 1939144704 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194747474747475, + "loss": 1.5622, + "theoretical_loss": 3.439260174743309, + "tokens_seen": 1939210240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194545454545455, + "loss": 1.5161, + "theoretical_loss": 3.439250424994988, + "tokens_seen": 1939275776 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041943434343434346, + "loss": 1.4962, + "theoretical_loss": 3.4392406756683958, + "tokens_seen": 1939341312 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194141414141414, + "loss": 1.4209, + "theoretical_loss": 3.4392309267635017, + "tokens_seen": 1939406848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193939393939394, + "loss": 1.2259, + "theoretical_loss": 3.439221178280272, + "tokens_seen": 1939472384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193737373737374, + "loss": 1.4304, + "theoretical_loss": 3.439211430218675, + "tokens_seen": 1939537920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193535353535354, + "loss": 1.4592, + "theoretical_loss": 3.439201682578678, + "tokens_seen": 1939603456 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041933333333333335, + "loss": 1.6319, + "theoretical_loss": 3.439191935360249, + "tokens_seen": 1939668992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193131313131313, + "loss": 1.351, + "theoretical_loss": 3.439182188563354, + "tokens_seen": 1939734528 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041929292929292927, + "loss": 1.6761, + "theoretical_loss": 3.4391724421879615, + "tokens_seen": 1939800064 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041927272727272733, + "loss": 1.5499, + "theoretical_loss": 3.4391626962340394, + "tokens_seen": 1939865600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004192525252525253, + "loss": 1.5205, + "theoretical_loss": 3.439152950701555, + "tokens_seen": 1939931136 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041923232323232325, + "loss": 1.4779, + "theoretical_loss": 3.439143205590475, + "tokens_seen": 1939996672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004192121212121212, + "loss": 1.4338, + "theoretical_loss": 3.439133460900768, + "tokens_seen": 1940062208 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.47330155968666077, + "objective/train/avg_token_score": 0.009856240823864937, + "objective/train/docs_used": 1096386, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5892785787582397, + "objective/train/original_loss": 4.016402721405029, + "objective/train/theoretical_loss": 3.439123716632401, + "objective/train/tokens_used": 298986976, + "objective/train/unlikelihood_loss": 0.07871534675359726, + "theoretical_loss": 3.439123716632401, + "tokens_seen": 1940127744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041919191919191916, + "loss": 1.7263, + "theoretical_loss": 3.439123716632401, + "tokens_seen": 1940127744 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191717171717172, + "loss": 1.4301, + "theoretical_loss": 3.439113972785342, + "tokens_seen": 1940193280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191515151515152, + "loss": 1.4758, + "theoretical_loss": 3.4391042293595584, + "tokens_seen": 1940258816 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041913131313131314, + "loss": 1.4596, + "theoretical_loss": 3.4390944863550175, + "tokens_seen": 1940324352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191111111111111, + "loss": 1.4683, + "theoretical_loss": 3.4390847437716863, + "tokens_seen": 1940389888 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041909090909090905, + "loss": 1.3133, + "theoretical_loss": 3.4390750016095337, + "tokens_seen": 1940455424 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190707070707071, + "loss": 1.6132, + "theoretical_loss": 3.439065259868527, + "tokens_seen": 1940520960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190505050505051, + "loss": 1.4538, + "theoretical_loss": 3.4390555185486327, + "tokens_seen": 1940586496 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041903030303030303, + "loss": 1.4229, + "theoretical_loss": 3.4390457776498193, + "tokens_seen": 1940652032 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419010101010101, + "loss": 1.363, + "theoretical_loss": 3.4390360371720545, + "tokens_seen": 1940717568 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418989898989899, + "loss": 1.4503, + "theoretical_loss": 3.439026297115305, + "tokens_seen": 1940783104 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418969696969697, + "loss": 1.3993, + "theoretical_loss": 3.4390165574795395, + "tokens_seen": 1940848640 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041894949494949497, + "loss": 1.4746, + "theoretical_loss": 3.4390068182647244, + "tokens_seen": 1940914176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189292929292929, + "loss": 1.5665, + "theoretical_loss": 3.4389970794708287, + "tokens_seen": 1940979712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189090909090909, + "loss": 1.6576, + "theoretical_loss": 3.4389873410978185, + "tokens_seen": 1941045248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188888888888889, + "loss": 1.4832, + "theoretical_loss": 3.4389776031456623, + "tokens_seen": 1941110784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188686868686869, + "loss": 1.1606, + "theoretical_loss": 3.4389678656143277, + "tokens_seen": 1941176320 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041884848484848486, + "loss": 1.5457, + "theoretical_loss": 3.4389581285037822, + "tokens_seen": 1941241856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188282828282828, + "loss": 1.2376, + "theoretical_loss": 3.4389483918139927, + "tokens_seen": 1941307392 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041880808080808083, + "loss": 1.5873, + "theoretical_loss": 3.438938655544928, + "tokens_seen": 1941372928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187878787878788, + "loss": 1.4951, + "theoretical_loss": 3.438928919696555, + "tokens_seen": 1941438464 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187676767676768, + "loss": 1.655, + "theoretical_loss": 3.4389191842688414, + "tokens_seen": 1941504000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041874747474747475, + "loss": 1.3632, + "theoretical_loss": 3.438909449261755, + "tokens_seen": 1941569536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187272727272727, + "loss": 1.4523, + "theoretical_loss": 3.4388997146752636, + "tokens_seen": 1941635072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187070707070707, + "loss": 1.3786, + "theoretical_loss": 3.4388899805093343, + "tokens_seen": 1941700608 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.43194037675857544, + "objective/train/avg_token_score": 0.020467616617679596, + "objective/train/docs_used": 1097742, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5595964193344116, + "objective/train/original_loss": 4.220158576965332, + "objective/train/theoretical_loss": 3.438880246763935, + "objective/train/tokens_used": 300625376, + "objective/train/unlikelihood_loss": 0.06359304487705231, + "theoretical_loss": 3.438880246763935, + "tokens_seen": 1941766144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186868686868687, + "loss": 1.4585, + "theoretical_loss": 3.438880246763935, + "tokens_seen": 1941766144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186666666666667, + "loss": 1.3775, + "theoretical_loss": 3.438870513439033, + "tokens_seen": 1941831680 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041864646464646464, + "loss": 1.4134, + "theoretical_loss": 3.438860780534597, + "tokens_seen": 1941897216 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041862626262626266, + "loss": 1.2068, + "theoretical_loss": 3.438851048050594, + "tokens_seen": 1941962752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186060606060606, + "loss": 1.6119, + "theoretical_loss": 3.4388413159869913, + "tokens_seen": 1942028288 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041858585858585857, + "loss": 1.4467, + "theoretical_loss": 3.438831584343757, + "tokens_seen": 1942093824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185656565656566, + "loss": 1.4658, + "theoretical_loss": 3.4388218531208583, + "tokens_seen": 1942159360 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041854545454545454, + "loss": 1.5447, + "theoretical_loss": 3.438812122318263, + "tokens_seen": 1942224896 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041852525252525255, + "loss": 1.5643, + "theoretical_loss": 3.4388023919359396, + "tokens_seen": 1942290432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185050505050505, + "loss": 1.305, + "theoretical_loss": 3.4387926619738547, + "tokens_seen": 1942355968 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041848484848484846, + "loss": 1.4774, + "theoretical_loss": 3.438782932431976, + "tokens_seen": 1942421504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184646464646465, + "loss": 1.4883, + "theoretical_loss": 3.438773203310272, + "tokens_seen": 1942487040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184444444444445, + "loss": 1.7094, + "theoretical_loss": 3.4387634746087103, + "tokens_seen": 1942552576 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041842424242424244, + "loss": 1.3692, + "theoretical_loss": 3.4387537463272575, + "tokens_seen": 1942618112 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184040404040404, + "loss": 1.5318, + "theoretical_loss": 3.438744018465882, + "tokens_seen": 1942683648 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041838383838383835, + "loss": 1.2879, + "theoretical_loss": 3.438734291024552, + "tokens_seen": 1942749184 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041836363636363637, + "loss": 1.6327, + "theoretical_loss": 3.438724564003235, + "tokens_seen": 1942814720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183434343434344, + "loss": 1.2888, + "theoretical_loss": 3.438714837401897, + "tokens_seen": 1942880256 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041832323232323233, + "loss": 1.5383, + "theoretical_loss": 3.4387051112205085, + "tokens_seen": 1942945792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183030303030303, + "loss": 1.4541, + "theoretical_loss": 3.4386953854590354, + "tokens_seen": 1943011328 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041828282828282825, + "loss": 1.4903, + "theoretical_loss": 3.4386856601174456, + "tokens_seen": 1943076864 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182626262626263, + "loss": 1.3577, + "theoretical_loss": 3.4386759351957066, + "tokens_seen": 1943142400 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041824242424242427, + "loss": 1.4549, + "theoretical_loss": 3.4386662106937873, + "tokens_seen": 1943207936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182222222222222, + "loss": 1.5026, + "theoretical_loss": 3.438656486611654, + "tokens_seen": 1943273472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182020202020202, + "loss": 1.4573, + "theoretical_loss": 3.4386467629492756, + "tokens_seen": 1943339008 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.38086509704589844, + "objective/train/avg_token_score": 0.017717378214001656, + "objective/train/docs_used": 1098461, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2602291107177734, + "objective/train/original_loss": 4.616687297821045, + "objective/train/theoretical_loss": 3.4386370397066193, + "objective/train/tokens_used": 302263776, + "objective/train/unlikelihood_loss": 0.05086660385131836, + "theoretical_loss": 3.4386370397066193, + "tokens_seen": 1943404544 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041818181818181814, + "loss": 1.4643, + "theoretical_loss": 3.4386370397066193, + "tokens_seen": 1943404544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181616161616162, + "loss": 1.4196, + "theoretical_loss": 3.4386273168836525, + "tokens_seen": 1943470080 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041814141414141416, + "loss": 1.3719, + "theoretical_loss": 3.4386175944803434, + "tokens_seen": 1943535616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181212121212121, + "loss": 1.6352, + "theoretical_loss": 3.4386078724966596, + "tokens_seen": 1943601152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181010101010101, + "loss": 1.4791, + "theoretical_loss": 3.438598150932569, + "tokens_seen": 1943666688 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041808080808080814, + "loss": 1.2838, + "theoretical_loss": 3.4385884297880387, + "tokens_seen": 1943732224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180606060606061, + "loss": 1.6374, + "theoretical_loss": 3.4385787090630373, + "tokens_seen": 1943797760 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041804040404040406, + "loss": 1.5597, + "theoretical_loss": 3.438568988757532, + "tokens_seen": 1943863296 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418020202020202, + "loss": 1.3555, + "theoretical_loss": 3.438559268871491, + "tokens_seen": 1943928832 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041799999999999997, + "loss": 1.5381, + "theoretical_loss": 3.4385495494048817, + "tokens_seen": 1943994368 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041797979797979803, + "loss": 1.6316, + "theoretical_loss": 3.4385398303576715, + "tokens_seen": 1944059904 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417959595959596, + "loss": 1.5205, + "theoretical_loss": 3.438530111729829, + "tokens_seen": 1944125440 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041793939393939395, + "loss": 1.4276, + "theoretical_loss": 3.4385203935213218, + "tokens_seen": 1944190976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179191919191919, + "loss": 1.4871, + "theoretical_loss": 3.4385106757321173, + "tokens_seen": 1944256512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178989898989899, + "loss": 1.406, + "theoretical_loss": 3.438500958362183, + "tokens_seen": 1944322048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178787878787879, + "loss": 1.6157, + "theoretical_loss": 3.438491241411487, + "tokens_seen": 1944387584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178585858585859, + "loss": 1.3643, + "theoretical_loss": 3.438481524879998, + "tokens_seen": 1944453120 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041783838383838384, + "loss": 1.5163, + "theoretical_loss": 3.438471808767683, + "tokens_seen": 1944518656 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178181818181818, + "loss": 1.5139, + "theoretical_loss": 3.4384620930745093, + "tokens_seen": 1944584192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177979797979798, + "loss": 1.6269, + "theoretical_loss": 3.438452377800445, + "tokens_seen": 1944649728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177777777777778, + "loss": 1.3211, + "theoretical_loss": 3.4384426629454587, + "tokens_seen": 1944715264 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177575757575758, + "loss": 1.4293, + "theoretical_loss": 3.438432948509517, + "tokens_seen": 1944780800 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041773737373737373, + "loss": 1.4411, + "theoretical_loss": 3.4384232344925882, + "tokens_seen": 1944846336 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041771717171717174, + "loss": 1.4302, + "theoretical_loss": 3.438413520894641, + "tokens_seen": 1944911872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176969696969697, + "loss": 1.5839, + "theoretical_loss": 3.438403807715641, + "tokens_seen": 1944977408 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.33687683939933777, + "objective/train/avg_token_score": 0.05385596305131912, + "objective/train/docs_used": 1099474, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2468277215957642, + "objective/train/original_loss": 4.72367000579834, + "objective/train/theoretical_loss": 3.4383940949555587, + "objective/train/tokens_used": 303902176, + "objective/train/unlikelihood_loss": 0.06228525936603546, + "theoretical_loss": 3.4383940949555587, + "tokens_seen": 1945042944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176767676767677, + "loss": 1.3905, + "theoretical_loss": 3.4383940949555587, + "tokens_seen": 1945042944 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041765656565656567, + "loss": 1.418, + "theoretical_loss": 3.43838438261436, + "tokens_seen": 1945108480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176363636363636, + "loss": 1.3343, + "theoretical_loss": 3.4383746706920135, + "tokens_seen": 1945174016 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041761616161616164, + "loss": 1.3567, + "theoretical_loss": 3.438364959188487, + "tokens_seen": 1945239552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175959595959596, + "loss": 1.467, + "theoretical_loss": 3.438355248103748, + "tokens_seen": 1945305088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175757575757576, + "loss": 1.5035, + "theoretical_loss": 3.4383455374377645, + "tokens_seen": 1945370624 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041755555555555556, + "loss": 1.5008, + "theoretical_loss": 3.4383358271905045, + "tokens_seen": 1945436160 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041753535353535357, + "loss": 1.2581, + "theoretical_loss": 3.4383261173619353, + "tokens_seen": 1945501696 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041751515151515153, + "loss": 1.6451, + "theoretical_loss": 3.438316407952026, + "tokens_seen": 1945567232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174949494949495, + "loss": 1.4594, + "theoretical_loss": 3.438306698960743, + "tokens_seen": 1945632768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174747474747475, + "loss": 1.5545, + "theoretical_loss": 3.4382969903880545, + "tokens_seen": 1945698304 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041745454545454545, + "loss": 1.6054, + "theoretical_loss": 3.4382872822339294, + "tokens_seen": 1945763840 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041743434343434347, + "loss": 1.6339, + "theoretical_loss": 3.4382775744983345, + "tokens_seen": 1945829376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174141414141414, + "loss": 1.3249, + "theoretical_loss": 3.4382678671812377, + "tokens_seen": 1945894912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173939393939394, + "loss": 1.4215, + "theoretical_loss": 3.438258160282607, + "tokens_seen": 1945960448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173737373737374, + "loss": 1.6104, + "theoretical_loss": 3.4382484538024105, + "tokens_seen": 1946025984 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041735353535353535, + "loss": 1.3356, + "theoretical_loss": 3.438238747740616, + "tokens_seen": 1946091520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041733333333333336, + "loss": 1.1932, + "theoretical_loss": 3.4382290420971913, + "tokens_seen": 1946157056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173131313131313, + "loss": 1.4409, + "theoretical_loss": 3.438219336872104, + "tokens_seen": 1946222592 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041729292929292927, + "loss": 1.5784, + "theoretical_loss": 3.4382096320653224, + "tokens_seen": 1946288128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172727272727273, + "loss": 1.7057, + "theoretical_loss": 3.4381999276768145, + "tokens_seen": 1946353664 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172525252525253, + "loss": 1.2891, + "theoretical_loss": 3.438190223706548, + "tokens_seen": 1946419200 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041723232323232325, + "loss": 1.4872, + "theoretical_loss": 3.43818052015449, + "tokens_seen": 1946484736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172121212121212, + "loss": 1.4779, + "theoretical_loss": 3.43817081702061, + "tokens_seen": 1946550272 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041719191919191916, + "loss": 1.3737, + "theoretical_loss": 3.4381611143048745, + "tokens_seen": 1946615808 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.4748900234699249, + "objective/train/avg_token_score": 0.007229257840663195, + "objective/train/docs_used": 1099984, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5736761093139648, + "objective/train/original_loss": 4.259389400482178, + "objective/train/theoretical_loss": 3.4381514120072523, + "objective/train/tokens_used": 305540576, + "objective/train/unlikelihood_loss": 0.05011461675167084, + "theoretical_loss": 3.4381514120072523, + "tokens_seen": 1946681344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171717171717172, + "loss": 1.6191, + "theoretical_loss": 3.4381514120072523, + "tokens_seen": 1946681344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171515151515152, + "loss": 1.5331, + "theoretical_loss": 3.4381417101277107, + "tokens_seen": 1946746880 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041713131313131314, + "loss": 1.5493, + "theoretical_loss": 3.438132008666218, + "tokens_seen": 1946812416 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171111111111111, + "loss": 1.3735, + "theoretical_loss": 3.438122307622742, + "tokens_seen": 1946877952 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041709090909090906, + "loss": 1.5348, + "theoretical_loss": 3.43811260699725, + "tokens_seen": 1946943488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170707070707071, + "loss": 1.5874, + "theoretical_loss": 3.4381029067897115, + "tokens_seen": 1947009024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170505050505051, + "loss": 1.4854, + "theoretical_loss": 3.438093207000093, + "tokens_seen": 1947074560 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041703030303030304, + "loss": 1.6018, + "theoretical_loss": 3.4380835076283622, + "tokens_seen": 1947140096 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417010101010101, + "loss": 1.3937, + "theoretical_loss": 3.4380738086744884, + "tokens_seen": 1947205632 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041698989898989895, + "loss": 1.3372, + "theoretical_loss": 3.4380641101384386, + "tokens_seen": 1947271168 + }, + { + "epoch": 0.17, + "learning_rate": 0.000416969696969697, + "loss": 1.4951, + "theoretical_loss": 3.4380544120201812, + "tokens_seen": 1947336704 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041694949494949497, + "loss": 1.2962, + "theoretical_loss": 3.438044714319684, + "tokens_seen": 1947402240 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041692929292929293, + "loss": 1.4141, + "theoretical_loss": 3.4380350170369143, + "tokens_seen": 1947467776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169090909090909, + "loss": 1.3446, + "theoretical_loss": 3.438025320171841, + "tokens_seen": 1947533312 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168888888888889, + "loss": 1.469, + "theoretical_loss": 3.438015623724432, + "tokens_seen": 1947598848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168686868686869, + "loss": 1.516, + "theoretical_loss": 3.4380059276946544, + "tokens_seen": 1947664384 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041684848484848486, + "loss": 1.4418, + "theoretical_loss": 3.437996232082477, + "tokens_seen": 1947729920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168282828282828, + "loss": 1.5376, + "theoretical_loss": 3.437986536887867, + "tokens_seen": 1947795456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168080808080808, + "loss": 1.396, + "theoretical_loss": 3.437976842110793, + "tokens_seen": 1947860992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167878787878788, + "loss": 1.2724, + "theoretical_loss": 3.4379671477512233, + "tokens_seen": 1947926528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167676767676768, + "loss": 1.306, + "theoretical_loss": 3.4379574538091244, + "tokens_seen": 1947992064 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041674747474747476, + "loss": 1.656, + "theoretical_loss": 3.437947760284466, + "tokens_seen": 1948057600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167272727272727, + "loss": 1.4577, + "theoretical_loss": 3.437938067177215, + "tokens_seen": 1948123136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167070707070707, + "loss": 1.3489, + "theoretical_loss": 3.43792837448734, + "tokens_seen": 1948188672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004166868686868687, + "loss": 1.574, + "theoretical_loss": 3.437918682214808, + "tokens_seen": 1948254208 + }, + { + "epoch": 0.17, + "objective/train/aligned_token_count": 0.3907013535499573, + "objective/train/avg_token_score": 0.006235840264707804, + "objective/train/docs_used": 1101410, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.310508370399475, + "objective/train/original_loss": 4.164927005767822, + "objective/train/theoretical_loss": 3.4379089903595883, + "objective/train/tokens_used": 307178976, + "objective/train/unlikelihood_loss": 0.08828544616699219, + "theoretical_loss": 3.4379089903595883, + "tokens_seen": 1948319744 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004166666666666667, + "loss": 1.4446, + "theoretical_loss": 3.4379089903595883, + "tokens_seen": 1948319744 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041664646464646465, + "loss": 1.258, + "theoretical_loss": 3.437899298921648, + "tokens_seen": 1948385280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166262626262626, + "loss": 1.2467, + "theoretical_loss": 3.4378896079009555, + "tokens_seen": 1948450816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166060606060606, + "loss": 1.2485, + "theoretical_loss": 3.437879917297479, + "tokens_seen": 1948516352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165858585858586, + "loss": 1.4547, + "theoretical_loss": 3.437870227111186, + "tokens_seen": 1948581888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165656565656566, + "loss": 1.5381, + "theoretical_loss": 3.4378605373420443, + "tokens_seen": 1948647424 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041654545454545454, + "loss": 1.4898, + "theoretical_loss": 3.4378508479900227, + "tokens_seen": 1948712960 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041652525252525255, + "loss": 1.6897, + "theoretical_loss": 3.437841159055089, + "tokens_seen": 1948778496 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165050505050505, + "loss": 1.5336, + "theoretical_loss": 3.4378314705372106, + "tokens_seen": 1948844032 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041648484848484847, + "loss": 1.4052, + "theoretical_loss": 3.437821782436356, + "tokens_seen": 1948909568 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164646464646465, + "loss": 1.5426, + "theoretical_loss": 3.4378120947524935, + "tokens_seen": 1948975104 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041644444444444444, + "loss": 1.5688, + "theoretical_loss": 3.4378024074855906, + "tokens_seen": 1949040640 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041642424242424245, + "loss": 1.5614, + "theoretical_loss": 3.437792720635616, + "tokens_seen": 1949106176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164040404040404, + "loss": 1.4658, + "theoretical_loss": 3.437783034202537, + "tokens_seen": 1949171712 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041638383838383836, + "loss": 1.407, + "theoretical_loss": 3.4377733481863215, + "tokens_seen": 1949237248 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041636363636363637, + "loss": 1.343, + "theoretical_loss": 3.4377636625869386, + "tokens_seen": 1949302784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163434343434344, + "loss": 1.5019, + "theoretical_loss": 3.4377539774043555, + "tokens_seen": 1949368320 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041632323232323234, + "loss": 1.4887, + "theoretical_loss": 3.4377442926385404, + "tokens_seen": 1949433856 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163030303030303, + "loss": 1.5129, + "theoretical_loss": 3.4377346082894618, + "tokens_seen": 1949499392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162828282828283, + "loss": 1.3991, + "theoretical_loss": 3.437724924357087, + "tokens_seen": 1949564928 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041626262626262626, + "loss": 1.6064, + "theoretical_loss": 3.437715240841385, + "tokens_seen": 1949630464 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162424242424243, + "loss": 1.5792, + "theoretical_loss": 3.437705557742323, + "tokens_seen": 1949696000 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041622222222222223, + "loss": 1.6014, + "theoretical_loss": 3.4376958750598696, + "tokens_seen": 1949761536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162020202020202, + "loss": 1.515, + "theoretical_loss": 3.437686192793992, + "tokens_seen": 1949827072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161818181818182, + "loss": 1.3131, + "theoretical_loss": 3.4376765109446596, + "tokens_seen": 1949892608 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.24596774578094482, + "objective/train/avg_token_score": 0.017320265993475914, + "objective/train/docs_used": 1102181, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8094052076339722, + "objective/train/original_loss": 4.91034460067749, + "objective/train/theoretical_loss": 3.4376668295118398, + "objective/train/tokens_used": 308817376, + "objective/train/unlikelihood_loss": 0.06164025887846947, + "theoretical_loss": 3.4376668295118398, + "tokens_seen": 1949958144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161616161616162, + "loss": 1.3284, + "theoretical_loss": 3.4376668295118398, + "tokens_seen": 1949958144 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041614141414141417, + "loss": 1.476, + "theoretical_loss": 3.4376571484955005, + "tokens_seen": 1950023680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161212121212121, + "loss": 1.4625, + "theoretical_loss": 3.4376474678956104, + "tokens_seen": 1950089216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161010101010101, + "loss": 1.2883, + "theoretical_loss": 3.437637787712137, + "tokens_seen": 1950154752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160808080808081, + "loss": 1.4984, + "theoretical_loss": 3.4376281079450486, + "tokens_seen": 1950220288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160606060606061, + "loss": 1.6164, + "theoretical_loss": 3.437618428594313, + "tokens_seen": 1950285824 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041604040404040406, + "loss": 1.4649, + "theoretical_loss": 3.4376087496598986, + "tokens_seen": 1950351360 + }, + { + "epoch": 0.18, + "learning_rate": 0.000416020202020202, + "loss": 1.5006, + "theoretical_loss": 3.437599071141774, + "tokens_seen": 1950416896 + }, + { + "epoch": 0.18, + "learning_rate": 0.000416, + "loss": 1.4269, + "theoretical_loss": 3.4375893930399064, + "tokens_seen": 1950482432 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041597979797979804, + "loss": 1.3195, + "theoretical_loss": 3.4375797153542647, + "tokens_seen": 1950547968 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415959595959596, + "loss": 1.5313, + "theoretical_loss": 3.437570038084816, + "tokens_seen": 1950613504 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041593939393939395, + "loss": 1.3833, + "theoretical_loss": 3.4375603612315295, + "tokens_seen": 1950679040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159191919191919, + "loss": 1.5523, + "theoretical_loss": 3.437550684794373, + "tokens_seen": 1950744576 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041589898989898987, + "loss": 1.3026, + "theoretical_loss": 3.4375410087733145, + "tokens_seen": 1950810112 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041587878787878793, + "loss": 1.1543, + "theoretical_loss": 3.437531333168322, + "tokens_seen": 1950875648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158585858585859, + "loss": 1.4297, + "theoretical_loss": 3.4375216579793637, + "tokens_seen": 1950941184 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041583838383838385, + "loss": 1.3946, + "theoretical_loss": 3.4375119832064076, + "tokens_seen": 1951006720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158181818181818, + "loss": 1.637, + "theoretical_loss": 3.4375023088494228, + "tokens_seen": 1951072256 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041579797979797976, + "loss": 1.4739, + "theoretical_loss": 3.4374926349083763, + "tokens_seen": 1951137792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157777777777778, + "loss": 1.1972, + "theoretical_loss": 3.4374829613832363, + "tokens_seen": 1951203328 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157575757575758, + "loss": 1.2674, + "theoretical_loss": 3.4374732882739716, + "tokens_seen": 1951268864 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041573737373737374, + "loss": 1.2435, + "theoretical_loss": 3.43746361558055, + "tokens_seen": 1951334400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157171717171717, + "loss": 1.283, + "theoretical_loss": 3.4374539433029394, + "tokens_seen": 1951399936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156969696969697, + "loss": 1.5095, + "theoretical_loss": 3.437444271441109, + "tokens_seen": 1951465472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156767676767677, + "loss": 1.5597, + "theoretical_loss": 3.4374345999950253, + "tokens_seen": 1951531008 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.5144794583320618, + "objective/train/avg_token_score": 0.006237310823053122, + "objective/train/docs_used": 1103575, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6771392822265625, + "objective/train/original_loss": 3.8277225494384766, + "objective/train/theoretical_loss": 3.437424928964658, + "objective/train/tokens_used": 310455776, + "objective/train/unlikelihood_loss": 0.050984472036361694, + "theoretical_loss": 3.437424928964658, + "tokens_seen": 1951596544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156565656565657, + "loss": 1.5589, + "theoretical_loss": 3.437424928964658, + "tokens_seen": 1951596544 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041563636363636363, + "loss": 1.3017, + "theoretical_loss": 3.437415258349975, + "tokens_seen": 1951662080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156161616161616, + "loss": 1.5017, + "theoretical_loss": 3.437405588150943, + "tokens_seen": 1951727616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155959595959596, + "loss": 1.3667, + "theoretical_loss": 3.437395918367532, + "tokens_seen": 1951793152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155757575757576, + "loss": 1.6398, + "theoretical_loss": 3.43738624899971, + "tokens_seen": 1951858688 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041555555555555557, + "loss": 1.5077, + "theoretical_loss": 3.437376580047444, + "tokens_seen": 1951924224 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155353535353535, + "loss": 1.6662, + "theoretical_loss": 3.437366911510703, + "tokens_seen": 1951989760 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041551515151515154, + "loss": 1.524, + "theoretical_loss": 3.437357243389455, + "tokens_seen": 1952055296 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154949494949495, + "loss": 1.2516, + "theoretical_loss": 3.437347575683668, + "tokens_seen": 1952120832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154747474747475, + "loss": 1.4622, + "theoretical_loss": 3.437337908393311, + "tokens_seen": 1952186368 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041545454545454546, + "loss": 1.5639, + "theoretical_loss": 3.437328241518352, + "tokens_seen": 1952251904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154343434343434, + "loss": 1.4582, + "theoretical_loss": 3.4373185750587583, + "tokens_seen": 1952317440 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041541414141414143, + "loss": 1.2993, + "theoretical_loss": 3.4373089090144986, + "tokens_seen": 1952382976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153939393939394, + "loss": 1.689, + "theoretical_loss": 3.4372992433855414, + "tokens_seen": 1952448512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153737373737374, + "loss": 1.5325, + "theoretical_loss": 3.4372895781718547, + "tokens_seen": 1952514048 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041535353535353535, + "loss": 1.5222, + "theoretical_loss": 3.437279913373407, + "tokens_seen": 1952579584 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041533333333333336, + "loss": 1.2857, + "theoretical_loss": 3.437270248990166, + "tokens_seen": 1952645120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153131313131313, + "loss": 1.4812, + "theoretical_loss": 3.4372605850221003, + "tokens_seen": 1952710656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152929292929293, + "loss": 1.4163, + "theoretical_loss": 3.437250921469178, + "tokens_seen": 1952776192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152727272727273, + "loss": 1.58, + "theoretical_loss": 3.437241258331367, + "tokens_seen": 1952841728 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041525252525252525, + "loss": 1.5932, + "theoretical_loss": 3.437231595608636, + "tokens_seen": 1952907264 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041523232323232326, + "loss": 1.3163, + "theoretical_loss": 3.4372219333009535, + "tokens_seen": 1952972800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152121212121212, + "loss": 1.4865, + "theoretical_loss": 3.437212271408287, + "tokens_seen": 1953038336 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041519191919191917, + "loss": 1.3725, + "theoretical_loss": 3.4372026099306057, + "tokens_seen": 1953103872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151717171717172, + "loss": 1.4658, + "theoretical_loss": 3.4371929488678767, + "tokens_seen": 1953169408 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.3589320480823517, + "objective/train/avg_token_score": 0.013197287917137146, + "objective/train/docs_used": 1104173, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2983180284500122, + "objective/train/original_loss": 4.760884761810303, + "objective/train/theoretical_loss": 3.4371832882200692, + "objective/train/tokens_used": 312094176, + "objective/train/unlikelihood_loss": 0.052073884755373, + "theoretical_loss": 3.4371832882200692, + "tokens_seen": 1953234944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151515151515152, + "loss": 1.3511, + "theoretical_loss": 3.4371832882200692, + "tokens_seen": 1953234944 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041513131313131315, + "loss": 1.3083, + "theoretical_loss": 3.4371736279871508, + "tokens_seen": 1953300480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151111111111111, + "loss": 1.6078, + "theoretical_loss": 3.43716396816909, + "tokens_seen": 1953366016 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041509090909090906, + "loss": 1.5163, + "theoretical_loss": 3.4371543087658556, + "tokens_seen": 1953431552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150707070707071, + "loss": 1.3672, + "theoretical_loss": 3.437144649777415, + "tokens_seen": 1953497088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150505050505051, + "loss": 1.3603, + "theoretical_loss": 3.4371349912037372, + "tokens_seen": 1953562624 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041503030303030304, + "loss": 1.472, + "theoretical_loss": 3.43712533304479, + "tokens_seen": 1953628160 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415010101010101, + "loss": 1.4868, + "theoretical_loss": 3.437115675300542, + "tokens_seen": 1953693696 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041498989898989896, + "loss": 1.3721, + "theoretical_loss": 3.4371060179709607, + "tokens_seen": 1953759232 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414969696969697, + "loss": 1.5007, + "theoretical_loss": 3.4370963610560157, + "tokens_seen": 1953824768 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414949494949495, + "loss": 1.3869, + "theoretical_loss": 3.437086704555674, + "tokens_seen": 1953890304 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041492929292929293, + "loss": 1.5926, + "theoretical_loss": 3.437077048469905, + "tokens_seen": 1953955840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149090909090909, + "loss": 1.5715, + "theoretical_loss": 3.437067392798676, + "tokens_seen": 1954021376 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041488888888888885, + "loss": 1.688, + "theoretical_loss": 3.4370577375419566, + "tokens_seen": 1954086912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148686868686869, + "loss": 1.3722, + "theoretical_loss": 3.4370480826997136, + "tokens_seen": 1954152448 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041484848484848487, + "loss": 1.4423, + "theoretical_loss": 3.437038428271916, + "tokens_seen": 1954217984 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041482828282828283, + "loss": 1.4912, + "theoretical_loss": 3.4370287742585326, + "tokens_seen": 1954283520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148080808080808, + "loss": 1.453, + "theoretical_loss": 3.4370191206595306, + "tokens_seen": 1954349056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147878787878788, + "loss": 1.473, + "theoretical_loss": 3.437009467474879, + "tokens_seen": 1954414592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147676767676768, + "loss": 1.533, + "theoretical_loss": 3.4369998147045466, + "tokens_seen": 1954480128 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041474747474747476, + "loss": 1.1881, + "theoretical_loss": 3.4369901623485006, + "tokens_seen": 1954545664 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147272727272727, + "loss": 1.4963, + "theoretical_loss": 3.43698051040671, + "tokens_seen": 1954611200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147070707070707, + "loss": 1.501, + "theoretical_loss": 3.4369708588791434, + "tokens_seen": 1954676736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146868686868687, + "loss": 1.4789, + "theoretical_loss": 3.4369612077657687, + "tokens_seen": 1954742272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146666666666667, + "loss": 1.4487, + "theoretical_loss": 3.4369515570665543, + "tokens_seen": 1954807808 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.42155423760414124, + "objective/train/avg_token_score": 0.007324489299207926, + "objective/train/docs_used": 1105519, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2957619428634644, + "objective/train/original_loss": 4.201072692871094, + "objective/train/theoretical_loss": 3.4369419067814686, + "objective/train/tokens_used": 313732576, + "objective/train/unlikelihood_loss": 0.06063661724328995, + "theoretical_loss": 3.4369419067814686, + "tokens_seen": 1954873344 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041464646464646466, + "loss": 1.3553, + "theoretical_loss": 3.4369419067814686, + "tokens_seen": 1954873344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146262626262626, + "loss": 1.3954, + "theoretical_loss": 3.4369322569104797, + "tokens_seen": 1954938880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146060606060606, + "loss": 1.4729, + "theoretical_loss": 3.4369226074535564, + "tokens_seen": 1955004416 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041458585858585863, + "loss": 1.5528, + "theoretical_loss": 3.436912958410667, + "tokens_seen": 1955069952 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145656565656566, + "loss": 1.2495, + "theoretical_loss": 3.436903309781779, + "tokens_seen": 1955135488 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041454545454545455, + "loss": 1.6581, + "theoretical_loss": 3.436893661566862, + "tokens_seen": 1955201024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145252525252525, + "loss": 1.3339, + "theoretical_loss": 3.4368840137658836, + "tokens_seen": 1955266560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145050505050505, + "loss": 1.4273, + "theoretical_loss": 3.436874366378812, + "tokens_seen": 1955332096 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041448484848484853, + "loss": 1.1655, + "theoretical_loss": 3.4368647194056168, + "tokens_seen": 1955397632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144646464646465, + "loss": 1.4596, + "theoretical_loss": 3.436855072846265, + "tokens_seen": 1955463168 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041444444444444444, + "loss": 1.295, + "theoretical_loss": 3.4368454267007253, + "tokens_seen": 1955528704 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041442424242424245, + "loss": 1.5501, + "theoretical_loss": 3.4368357809689667, + "tokens_seen": 1955594240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144040404040404, + "loss": 1.4875, + "theoretical_loss": 3.436826135650957, + "tokens_seen": 1955659776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143838383838384, + "loss": 1.4391, + "theoretical_loss": 3.436816490746665, + "tokens_seen": 1955725312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143636363636364, + "loss": 1.436, + "theoretical_loss": 3.436806846256058, + "tokens_seen": 1955790848 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041434343434343433, + "loss": 1.6272, + "theoretical_loss": 3.436797202179106, + "tokens_seen": 1955856384 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041432323232323235, + "loss": 1.5892, + "theoretical_loss": 3.4367875585157766, + "tokens_seen": 1955921920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143030303030303, + "loss": 1.5377, + "theoretical_loss": 3.4367779152660383, + "tokens_seen": 1955987456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142828282828283, + "loss": 1.1497, + "theoretical_loss": 3.436768272429859, + "tokens_seen": 1956052992 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041426262626262627, + "loss": 1.3424, + "theoretical_loss": 3.4367586300072075, + "tokens_seen": 1956118528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142424242424243, + "loss": 1.6651, + "theoretical_loss": 3.4367489879980524, + "tokens_seen": 1956184064 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041422222222222224, + "loss": 1.3298, + "theoretical_loss": 3.436739346402362, + "tokens_seen": 1956249600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142020202020202, + "loss": 1.48, + "theoretical_loss": 3.436729705220105, + "tokens_seen": 1956315136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141818181818182, + "loss": 1.5618, + "theoretical_loss": 3.436720064451249, + "tokens_seen": 1956380672 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041416161616161616, + "loss": 1.265, + "theoretical_loss": 3.4367104240957635, + "tokens_seen": 1956446208 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.32251954078674316, + "objective/train/avg_token_score": 0.009885277599096298, + "objective/train/docs_used": 1106102, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.99889075756073, + "objective/train/original_loss": 4.139228820800781, + "objective/train/theoretical_loss": 3.436700784153616, + "objective/train/tokens_used": 315370976, + "objective/train/unlikelihood_loss": 0.07105885446071625, + "theoretical_loss": 3.436700784153616, + "tokens_seen": 1956511744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141414141414142, + "loss": 1.3508, + "theoretical_loss": 3.436700784153616, + "tokens_seen": 1956511744 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041412121212121213, + "loss": 1.4683, + "theoretical_loss": 3.4366911446247754, + "tokens_seen": 1956577280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141010101010101, + "loss": 1.5079, + "theoretical_loss": 3.43668150550921, + "tokens_seen": 1956642816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140808080808081, + "loss": 1.5571, + "theoretical_loss": 3.4366718668068885, + "tokens_seen": 1956708352 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041406060606060606, + "loss": 1.4941, + "theoretical_loss": 3.436662228517779, + "tokens_seen": 1956773888 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041404040404040407, + "loss": 1.5562, + "theoretical_loss": 3.4366525906418497, + "tokens_seen": 1956839424 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414020202020202, + "loss": 1.7608, + "theoretical_loss": 3.43664295317907, + "tokens_seen": 1956904960 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414, + "loss": 1.1626, + "theoretical_loss": 3.4366333161294076, + "tokens_seen": 1956970496 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413979797979798, + "loss": 1.2521, + "theoretical_loss": 3.436623679492831, + "tokens_seen": 1957036032 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413959595959596, + "loss": 1.5362, + "theoretical_loss": 3.436614043269309, + "tokens_seen": 1957101568 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041393939393939396, + "loss": 1.3286, + "theoretical_loss": 3.43660440745881, + "tokens_seen": 1957167104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139191919191919, + "loss": 1.4399, + "theoretical_loss": 3.4365947720613015, + "tokens_seen": 1957232640 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041389898989898987, + "loss": 1.478, + "theoretical_loss": 3.4365851370767535, + "tokens_seen": 1957298176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138787878787879, + "loss": 1.4755, + "theoretical_loss": 3.436575502505134, + "tokens_seen": 1957363712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138585858585859, + "loss": 1.505, + "theoretical_loss": 3.4365658683464106, + "tokens_seen": 1957429248 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041383838383838385, + "loss": 1.4277, + "theoretical_loss": 3.4365562346005527, + "tokens_seen": 1957494784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138181818181818, + "loss": 1.2607, + "theoretical_loss": 3.436546601267529, + "tokens_seen": 1957560320 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041379797979797977, + "loss": 1.4686, + "theoretical_loss": 3.4365369683473066, + "tokens_seen": 1957625856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041377777777777783, + "loss": 1.1683, + "theoretical_loss": 3.4365273358398554, + "tokens_seen": 1957691392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137575757575758, + "loss": 1.3763, + "theoretical_loss": 3.4365177037451433, + "tokens_seen": 1957756928 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041373737373737374, + "loss": 1.1928, + "theoretical_loss": 3.4365080720631394, + "tokens_seen": 1957822464 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137171717171717, + "loss": 1.5801, + "theoretical_loss": 3.4364984407938115, + "tokens_seen": 1957888000 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041369696969696966, + "loss": 1.2228, + "theoretical_loss": 3.436488809937128, + "tokens_seen": 1957953536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136767676767677, + "loss": 1.3379, + "theoretical_loss": 3.4364791794930576, + "tokens_seen": 1958019072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136565656565657, + "loss": 1.3664, + "theoretical_loss": 3.4364695494615694, + "tokens_seen": 1958084608 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.4745234549045563, + "objective/train/avg_token_score": 0.011926386505365372, + "objective/train/docs_used": 1106840, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.578378438949585, + "objective/train/original_loss": 4.3692216873168945, + "objective/train/theoretical_loss": 3.4364599198426307, + "objective/train/tokens_used": 317009376, + "objective/train/unlikelihood_loss": 0.04125186428427696, + "theoretical_loss": 3.4364599198426307, + "tokens_seen": 1958150144 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041363636363636364, + "loss": 1.6483, + "theoretical_loss": 3.4364599198426307, + "tokens_seen": 1958150144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136161616161616, + "loss": 1.5616, + "theoretical_loss": 3.4364502906362118, + "tokens_seen": 1958215680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135959595959596, + "loss": 1.3994, + "theoretical_loss": 3.4364406618422794, + "tokens_seen": 1958281216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135757575757576, + "loss": 1.3587, + "theoretical_loss": 3.436431033460803, + "tokens_seen": 1958346752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135555555555556, + "loss": 1.526, + "theoretical_loss": 3.4364214054917515, + "tokens_seen": 1958412288 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041353535353535353, + "loss": 1.2816, + "theoretical_loss": 3.436411777935092, + "tokens_seen": 1958477824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135151515151515, + "loss": 1.3146, + "theoretical_loss": 3.4364021507907943, + "tokens_seen": 1958543360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134949494949495, + "loss": 1.3607, + "theoretical_loss": 3.4363925240588267, + "tokens_seen": 1958608896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134747474747475, + "loss": 1.4933, + "theoretical_loss": 3.4363828977391577, + "tokens_seen": 1958674432 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041345454545454547, + "loss": 1.4624, + "theoretical_loss": 3.436373271831755, + "tokens_seen": 1958739968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134343434343434, + "loss": 1.4388, + "theoretical_loss": 3.4363636463365888, + "tokens_seen": 1958805504 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041341414141414143, + "loss": 1.3727, + "theoretical_loss": 3.436354021253626, + "tokens_seen": 1958871040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133939393939394, + "loss": 1.3955, + "theoretical_loss": 3.436344396582837, + "tokens_seen": 1958936576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133737373737374, + "loss": 1.3918, + "theoretical_loss": 3.436334772324188, + "tokens_seen": 1959002112 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041335353535353536, + "loss": 1.4894, + "theoretical_loss": 3.43632514847765, + "tokens_seen": 1959067648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133333333333333, + "loss": 1.4914, + "theoretical_loss": 3.4363155250431894, + "tokens_seen": 1959133184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133131313131313, + "loss": 1.487, + "theoretical_loss": 3.4363059020207762, + "tokens_seen": 1959198720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132929292929293, + "loss": 1.4438, + "theoretical_loss": 3.436296279410379, + "tokens_seen": 1959264256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132727272727273, + "loss": 1.4592, + "theoretical_loss": 3.436286657211965, + "tokens_seen": 1959329792 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041325252525252525, + "loss": 1.5674, + "theoretical_loss": 3.4362770354255043, + "tokens_seen": 1959395328 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041323232323232326, + "loss": 1.5201, + "theoretical_loss": 3.436267414050965, + "tokens_seen": 1959460864 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132121212121212, + "loss": 1.3334, + "theoretical_loss": 3.4362577930883154, + "tokens_seen": 1959526400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131919191919192, + "loss": 1.6545, + "theoretical_loss": 3.436248172537524, + "tokens_seen": 1959591936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131717171717172, + "loss": 1.5059, + "theoretical_loss": 3.43623855239856, + "tokens_seen": 1959657472 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041315151515151514, + "loss": 1.4031, + "theoretical_loss": 3.4362289326713915, + "tokens_seen": 1959723008 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.34634652733802795, + "objective/train/avg_token_score": 0.02731332927942276, + "objective/train/docs_used": 1108280, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.078272819519043, + "objective/train/original_loss": 4.436617851257324, + "objective/train/theoretical_loss": 3.4362193133559877, + "objective/train/tokens_used": 318647776, + "objective/train/unlikelihood_loss": 0.09181275963783264, + "theoretical_loss": 3.4362193133559877, + "tokens_seen": 1959788544 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041313131313131315, + "loss": 1.3301, + "theoretical_loss": 3.4362193133559877, + "tokens_seen": 1959788544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131111111111111, + "loss": 1.2653, + "theoretical_loss": 3.4362096944523164, + "tokens_seen": 1959854080 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041309090909090907, + "loss": 1.5523, + "theoretical_loss": 3.436200075960347, + "tokens_seen": 1959919616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130707070707071, + "loss": 1.5079, + "theoretical_loss": 3.4361904578800466, + "tokens_seen": 1959985152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130505050505051, + "loss": 1.3166, + "theoretical_loss": 3.436180840211386, + "tokens_seen": 1960050688 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041303030303030305, + "loss": 1.5395, + "theoretical_loss": 3.436171222954332, + "tokens_seen": 1960116224 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413010101010101, + "loss": 1.2939, + "theoretical_loss": 3.436161606108855, + "tokens_seen": 1960181760 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041298989898989896, + "loss": 1.4107, + "theoretical_loss": 3.436151989674922, + "tokens_seen": 1960247296 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041296969696969697, + "loss": 1.3592, + "theoretical_loss": 3.436142373652502, + "tokens_seen": 1960312832 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412949494949495, + "loss": 1.5549, + "theoretical_loss": 3.4361327580415644, + "tokens_seen": 1960378368 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041292929292929294, + "loss": 1.5716, + "theoretical_loss": 3.436123142842077, + "tokens_seen": 1960443904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129090909090909, + "loss": 1.3391, + "theoretical_loss": 3.4361135280540087, + "tokens_seen": 1960509440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128888888888889, + "loss": 1.4607, + "theoretical_loss": 3.436103913677328, + "tokens_seen": 1960574976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128686868686869, + "loss": 1.35, + "theoretical_loss": 3.4360942997120043, + "tokens_seen": 1960640512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128484848484849, + "loss": 1.1904, + "theoretical_loss": 3.436084686158005, + "tokens_seen": 1960706048 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041282828282828283, + "loss": 1.216, + "theoretical_loss": 3.4360750730153002, + "tokens_seen": 1960771584 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128080808080808, + "loss": 1.3311, + "theoretical_loss": 3.4360654602838574, + "tokens_seen": 1960837120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127878787878788, + "loss": 1.4853, + "theoretical_loss": 3.4360558479636456, + "tokens_seen": 1960902656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127676767676768, + "loss": 1.6454, + "theoretical_loss": 3.4360462360546333, + "tokens_seen": 1960968192 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041274747474747477, + "loss": 1.2993, + "theoretical_loss": 3.43603662455679, + "tokens_seen": 1961033728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127272727272727, + "loss": 1.2265, + "theoretical_loss": 3.436027013470083, + "tokens_seen": 1961099264 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127070707070707, + "loss": 1.3192, + "theoretical_loss": 3.436017402794482, + "tokens_seen": 1961164800 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041268686868686875, + "loss": 1.6334, + "theoretical_loss": 3.4360077925299555, + "tokens_seen": 1961230336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126666666666667, + "loss": 1.3333, + "theoretical_loss": 3.435998182676472, + "tokens_seen": 1961295872 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041264646464646466, + "loss": 1.3261, + "theoretical_loss": 3.4359885732340008, + "tokens_seen": 1961361408 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.43071848154067993, + "objective/train/avg_token_score": 0.014302323572337627, + "objective/train/docs_used": 1108943, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3354324102401733, + "objective/train/original_loss": 4.234708309173584, + "objective/train/theoretical_loss": 3.4359789642025094, + "objective/train/tokens_used": 320286176, + "objective/train/unlikelihood_loss": 0.055439990013837814, + "theoretical_loss": 3.4359789642025094, + "tokens_seen": 1961426944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126262626262626, + "loss": 1.5046, + "theoretical_loss": 3.4359789642025094, + "tokens_seen": 1961426944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126060606060606, + "loss": 1.2966, + "theoretical_loss": 3.4359693555819675, + "tokens_seen": 1961492480 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041258585858585864, + "loss": 1.4952, + "theoretical_loss": 3.435959747372343, + "tokens_seen": 1961558016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125656565656566, + "loss": 1.3245, + "theoretical_loss": 3.4359501395736056, + "tokens_seen": 1961623552 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041254545454545455, + "loss": 1.3205, + "theoretical_loss": 3.4359405321857235, + "tokens_seen": 1961689088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125252525252525, + "loss": 1.2586, + "theoretical_loss": 3.4359309252086647, + "tokens_seen": 1961754624 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041250505050505047, + "loss": 1.124, + "theoretical_loss": 3.435921318642399, + "tokens_seen": 1961820160 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041248484848484853, + "loss": 1.2376, + "theoretical_loss": 3.4359117124868948, + "tokens_seen": 1961885696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124646464646465, + "loss": 1.5335, + "theoretical_loss": 3.4359021067421205, + "tokens_seen": 1961951232 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041244444444444445, + "loss": 1.5446, + "theoretical_loss": 3.435892501408045, + "tokens_seen": 1962016768 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124242424242424, + "loss": 1.2378, + "theoretical_loss": 3.435882896484637, + "tokens_seen": 1962082304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124040404040404, + "loss": 1.3928, + "theoretical_loss": 3.4358732919718653, + "tokens_seen": 1962147840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123838383838384, + "loss": 1.5574, + "theoretical_loss": 3.435863687869699, + "tokens_seen": 1962213376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123636363636364, + "loss": 1.2501, + "theoretical_loss": 3.4358540841781062, + "tokens_seen": 1962278912 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041234343434343434, + "loss": 1.3118, + "theoretical_loss": 3.435844480897056, + "tokens_seen": 1962344448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123232323232323, + "loss": 1.3476, + "theoretical_loss": 3.435834878026516, + "tokens_seen": 1962409984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123030303030303, + "loss": 1.2193, + "theoretical_loss": 3.435825275566457, + "tokens_seen": 1962475520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122828282828283, + "loss": 1.7092, + "theoretical_loss": 3.4358156735168466, + "tokens_seen": 1962541056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122626262626263, + "loss": 1.3348, + "theoretical_loss": 3.4358060718776535, + "tokens_seen": 1962606592 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041224242424242423, + "loss": 1.4579, + "theoretical_loss": 3.435796470648846, + "tokens_seen": 1962672128 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041222222222222224, + "loss": 1.4916, + "theoretical_loss": 3.4357868698303946, + "tokens_seen": 1962737664 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122020202020202, + "loss": 1.5911, + "theoretical_loss": 3.435777269422266, + "tokens_seen": 1962803200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121818181818182, + "loss": 1.3839, + "theoretical_loss": 3.4357676694244303, + "tokens_seen": 1962868736 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041216161616161617, + "loss": 1.4081, + "theoretical_loss": 3.4357580698368557, + "tokens_seen": 1962934272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121414141414141, + "loss": 1.2923, + "theoretical_loss": 3.435748470659511, + "tokens_seen": 1962999808 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.3285679519176483, + "objective/train/avg_token_score": 0.03589082136750221, + "objective/train/docs_used": 1110285, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.1867942810058594, + "objective/train/original_loss": 5.2249436378479, + "objective/train/theoretical_loss": 3.4357388718923656, + "objective/train/tokens_used": 321924576, + "objective/train/unlikelihood_loss": 0.03941744193434715, + "theoretical_loss": 3.4357388718923656, + "tokens_seen": 1963065344 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041212121212121214, + "loss": 1.4527, + "theoretical_loss": 3.4357388718923656, + "tokens_seen": 1963065344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121010101010101, + "loss": 1.3501, + "theoretical_loss": 3.4357292735353875, + "tokens_seen": 1963130880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004120808080808081, + "loss": 1.4519, + "theoretical_loss": 3.4357196755885457, + "tokens_seen": 1963196416 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041206060606060606, + "loss": 1.4366, + "theoretical_loss": 3.435710078051809, + "tokens_seen": 1963261952 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041204040404040407, + "loss": 1.3558, + "theoretical_loss": 3.435700480925146, + "tokens_seen": 1963327488 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041202020202020203, + "loss": 1.6353, + "theoretical_loss": 3.435690884208526, + "tokens_seen": 1963393024 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412, + "loss": 1.477, + "theoretical_loss": 3.4356812879019176, + "tokens_seen": 1963458560 + }, + { + "epoch": 0.18, + "learning_rate": 0.000411979797979798, + "loss": 1.6616, + "theoretical_loss": 3.4356716920052897, + "tokens_seen": 1963524096 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041195959595959595, + "loss": 1.2516, + "theoretical_loss": 3.4356620965186107, + "tokens_seen": 1963589632 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041193939393939396, + "loss": 1.4148, + "theoretical_loss": 3.435652501441849, + "tokens_seen": 1963655168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119191919191919, + "loss": 1.4648, + "theoretical_loss": 3.4356429067749747, + "tokens_seen": 1963720704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118989898989899, + "loss": 1.4111, + "theoretical_loss": 3.4356333125179557, + "tokens_seen": 1963786240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118787878787879, + "loss": 1.4834, + "theoretical_loss": 3.435623718670761, + "tokens_seen": 1963851776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118585858585859, + "loss": 1.4329, + "theoretical_loss": 3.43561412523336, + "tokens_seen": 1963917312 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041183838383838386, + "loss": 1.4315, + "theoretical_loss": 3.4356045322057205, + "tokens_seen": 1963982848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118181818181818, + "loss": 1.1834, + "theoretical_loss": 3.435594939587812, + "tokens_seen": 1964048384 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041179797979797977, + "loss": 1.1604, + "theoretical_loss": 3.435585347379603, + "tokens_seen": 1964113920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117777777777778, + "loss": 1.3149, + "theoretical_loss": 3.4355757555810627, + "tokens_seen": 1964179456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117575757575758, + "loss": 1.4457, + "theoretical_loss": 3.435566164192159, + "tokens_seen": 1964244992 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041173737373737375, + "loss": 1.4566, + "theoretical_loss": 3.435556573212862, + "tokens_seen": 1964310528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117171717171717, + "loss": 1.4909, + "theoretical_loss": 3.4355469826431406, + "tokens_seen": 1964376064 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041169696969696966, + "loss": 1.3193, + "theoretical_loss": 3.435537392482962, + "tokens_seen": 1964441600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041167676767676773, + "loss": 1.5391, + "theoretical_loss": 3.4355278027322966, + "tokens_seen": 1964507136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116565656565657, + "loss": 1.4511, + "theoretical_loss": 3.435518213391113, + "tokens_seen": 1964572672 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041163636363636364, + "loss": 1.4846, + "theoretical_loss": 3.4355086244593793, + "tokens_seen": 1964638208 + }, + { + "epoch": 0.18, + "objective/train/aligned_token_count": 0.3200146555900574, + "objective/train/avg_token_score": 0.009661064483225346, + "objective/train/docs_used": 1110737, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.0394518375396729, + "objective/train/original_loss": 3.6994714736938477, + "objective/train/theoretical_loss": 3.435499035937065, + "objective/train/tokens_used": 323562976, + "objective/train/unlikelihood_loss": 0.08720573782920837, + "theoretical_loss": 3.435499035937065, + "tokens_seen": 1964703744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116161616161616, + "loss": 1.2846, + "theoretical_loss": 3.435499035937065, + "tokens_seen": 1964703744 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041159595959595956, + "loss": 1.4118, + "theoretical_loss": 3.435489447824139, + "tokens_seen": 1964769280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115757575757576, + "loss": 1.3332, + "theoretical_loss": 3.43547986012057, + "tokens_seen": 1964834816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115555555555556, + "loss": 1.371, + "theoretical_loss": 3.4354702728263264, + "tokens_seen": 1964900352 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041153535353535354, + "loss": 1.5411, + "theoretical_loss": 3.435460685941378, + "tokens_seen": 1964965888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115151515151515, + "loss": 1.4533, + "theoretical_loss": 3.435451099465693, + "tokens_seen": 1965031424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114949494949495, + "loss": 1.2664, + "theoretical_loss": 3.4354415133992404, + "tokens_seen": 1965096960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114747474747475, + "loss": 1.4433, + "theoretical_loss": 3.4354319277419894, + "tokens_seen": 1965162496 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041145454545454547, + "loss": 1.4658, + "theoretical_loss": 3.4354223424939088, + "tokens_seen": 1965228032 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041143434343434343, + "loss": 1.382, + "theoretical_loss": 3.435412757654967, + "tokens_seen": 1965293568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114141414141414, + "loss": 1.3177, + "theoretical_loss": 3.4354031732251333, + "tokens_seen": 1965359104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004113939393939394, + "loss": 1.2478, + "theoretical_loss": 3.4353935892043768, + "tokens_seen": 1965424640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004113737373737374, + "loss": 1.7692, + "theoretical_loss": 3.435384005592666, + "tokens_seen": 1965490176 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041135353535353536, + "loss": 1.5251, + "theoretical_loss": 3.4353744223899696, + "tokens_seen": 1965555712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004113333333333333, + "loss": 1.441, + "theoretical_loss": 3.4353648395962573, + "tokens_seen": 1965621248 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041131313131313133, + "loss": 1.3334, + "theoretical_loss": 3.435355257211498, + "tokens_seen": 1965686784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112929292929293, + "loss": 1.4239, + "theoretical_loss": 3.4353456752356593, + "tokens_seen": 1965752320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112727272727273, + "loss": 1.3418, + "theoretical_loss": 3.4353360936687114, + "tokens_seen": 1965817856 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041125252525252526, + "loss": 1.1314, + "theoretical_loss": 3.4353265125106223, + "tokens_seen": 1965883392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112323232323232, + "loss": 1.4341, + "theoretical_loss": 3.4353169317613617, + "tokens_seen": 1965948928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112121212121212, + "loss": 1.5507, + "theoretical_loss": 3.4353073514208985, + "tokens_seen": 1966014464 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041119191919191924, + "loss": 1.3749, + "theoretical_loss": 3.4352977714892017, + "tokens_seen": 1966080000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111717171717172, + "loss": 1.5129, + "theoretical_loss": 3.4352881919662392, + "tokens_seen": 1966145536 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041115151515151515, + "loss": 1.382, + "theoretical_loss": 3.435278612851981, + "tokens_seen": 1966211072 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041113131313131316, + "loss": 1.6181, + "theoretical_loss": 3.435269034146396, + "tokens_seen": 1966276608 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.4528347849845886, + "objective/train/avg_token_score": 0.006985062267631292, + "objective/train/docs_used": 1112038, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.5920250415802002, + "objective/train/original_loss": 4.1393818855285645, + "objective/train/theoretical_loss": 3.4352594558494527, + "objective/train/tokens_used": 325201376, + "objective/train/unlikelihood_loss": 0.0694524422287941, + "theoretical_loss": 3.4352594558494527, + "tokens_seen": 1966342144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111111111111111, + "loss": 1.3822, + "theoretical_loss": 3.4352594558494527, + "tokens_seen": 1966342144 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041109090909090913, + "loss": 1.7697, + "theoretical_loss": 3.4352498779611196, + "tokens_seen": 1966407680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110707070707071, + "loss": 1.4781, + "theoretical_loss": 3.435240300481367, + "tokens_seen": 1966473216 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041105050505050504, + "loss": 1.381, + "theoretical_loss": 3.4352307234101627, + "tokens_seen": 1966538752 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041103030303030305, + "loss": 1.4222, + "theoretical_loss": 3.4352211467474767, + "tokens_seen": 1966604288 + }, + { + "epoch": 0.19, + "learning_rate": 0.000411010101010101, + "loss": 1.5048, + "theoretical_loss": 3.4352115704932764, + "tokens_seen": 1966669824 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410989898989899, + "loss": 1.3564, + "theoretical_loss": 3.4352019946475325, + "tokens_seen": 1966735360 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410969696969697, + "loss": 1.1223, + "theoretical_loss": 3.435192419210213, + "tokens_seen": 1966800896 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410949494949495, + "loss": 1.5934, + "theoretical_loss": 3.435182844181287, + "tokens_seen": 1966866432 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041092929292929295, + "loss": 1.2154, + "theoretical_loss": 3.435173269560723, + "tokens_seen": 1966931968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109090909090909, + "loss": 1.4263, + "theoretical_loss": 3.435163695348491, + "tokens_seen": 1966997504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108888888888889, + "loss": 1.4043, + "theoretical_loss": 3.435154121544559, + "tokens_seen": 1967063040 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041086868686868687, + "loss": 1.263, + "theoretical_loss": 3.4351445481488967, + "tokens_seen": 1967128576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108484848484849, + "loss": 1.5556, + "theoretical_loss": 3.435134975161473, + "tokens_seen": 1967194112 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041082828282828284, + "loss": 1.6561, + "theoretical_loss": 3.4351254025822566, + "tokens_seen": 1967259648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108080808080808, + "loss": 1.3542, + "theoretical_loss": 3.435115830411217, + "tokens_seen": 1967325184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107878787878788, + "loss": 1.328, + "theoretical_loss": 3.4351062586483225, + "tokens_seen": 1967390720 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041076767676767676, + "loss": 1.6009, + "theoretical_loss": 3.4350966872935422, + "tokens_seen": 1967456256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107474747474748, + "loss": 1.3577, + "theoretical_loss": 3.4350871163468457, + "tokens_seen": 1967521792 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041072727272727273, + "loss": 1.3894, + "theoretical_loss": 3.435077545808202, + "tokens_seen": 1967587328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107070707070707, + "loss": 1.3641, + "theoretical_loss": 3.435067975677579, + "tokens_seen": 1967652864 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106868686868687, + "loss": 1.4913, + "theoretical_loss": 3.4350584059549467, + "tokens_seen": 1967718400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106666666666667, + "loss": 1.6145, + "theoretical_loss": 3.435048836640274, + "tokens_seen": 1967783936 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041064646464646467, + "loss": 1.5496, + "theoretical_loss": 3.4350392677335297, + "tokens_seen": 1967849472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106262626262626, + "loss": 1.4591, + "theoretical_loss": 3.435029699234683, + "tokens_seen": 1967915008 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.3913734257221222, + "objective/train/avg_token_score": 0.011882202699780464, + "objective/train/docs_used": 1112624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2495681047439575, + "objective/train/original_loss": 4.346351623535156, + "objective/train/theoretical_loss": 3.435020131143703, + "objective/train/tokens_used": 326839776, + "objective/train/unlikelihood_loss": 0.07031120359897614, + "theoretical_loss": 3.435020131143703, + "tokens_seen": 1967980544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106060606060606, + "loss": 1.2495, + "theoretical_loss": 3.435020131143703, + "tokens_seen": 1967980544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105858585858586, + "loss": 1.5182, + "theoretical_loss": 3.4350105634605583, + "tokens_seen": 1968046080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105656565656566, + "loss": 1.4065, + "theoretical_loss": 3.435000996185219, + "tokens_seen": 1968111616 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041054545454545456, + "loss": 1.4003, + "theoretical_loss": 3.4349914293176527, + "tokens_seen": 1968177152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105252525252525, + "loss": 1.4241, + "theoretical_loss": 3.4349818628578292, + "tokens_seen": 1968242688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105050505050505, + "loss": 1.5984, + "theoretical_loss": 3.4349722968057175, + "tokens_seen": 1968308224 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041048484848484854, + "loss": 1.6424, + "theoretical_loss": 3.4349627311612867, + "tokens_seen": 1968373760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104646464646465, + "loss": 1.5381, + "theoretical_loss": 3.434953165924506, + "tokens_seen": 1968439296 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041044444444444445, + "loss": 1.2385, + "theoretical_loss": 3.434943601095344, + "tokens_seen": 1968504832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104242424242424, + "loss": 1.2404, + "theoretical_loss": 3.43493403667377, + "tokens_seen": 1968570368 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041040404040404037, + "loss": 1.4081, + "theoretical_loss": 3.4349244726597528, + "tokens_seen": 1968635904 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041038383838383843, + "loss": 1.4257, + "theoretical_loss": 3.434914909053262, + "tokens_seen": 1968701440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103636363636364, + "loss": 1.2265, + "theoretical_loss": 3.434905345854266, + "tokens_seen": 1968766976 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041034343434343435, + "loss": 1.258, + "theoretical_loss": 3.4348957830627347, + "tokens_seen": 1968832512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103232323232323, + "loss": 1.4968, + "theoretical_loss": 3.434886220678637, + "tokens_seen": 1968898048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103030303030303, + "loss": 1.4694, + "theoretical_loss": 3.434876658701941, + "tokens_seen": 1968963584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102828282828283, + "loss": 1.5733, + "theoretical_loss": 3.434867097132617, + "tokens_seen": 1969029120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102626262626263, + "loss": 1.5411, + "theoretical_loss": 3.4348575359706333, + "tokens_seen": 1969094656 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041024242424242424, + "loss": 1.4478, + "theoretical_loss": 3.43484797521596, + "tokens_seen": 1969160192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102222222222222, + "loss": 1.3074, + "theoretical_loss": 3.4348384148685644, + "tokens_seen": 1969225728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102020202020202, + "loss": 1.2974, + "theoretical_loss": 3.4348288549284174, + "tokens_seen": 1969291264 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101818181818182, + "loss": 1.3546, + "theoretical_loss": 3.434819295395487, + "tokens_seen": 1969356800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101616161616162, + "loss": 1.1625, + "theoretical_loss": 3.434809736269743, + "tokens_seen": 1969422336 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041014141414141413, + "loss": 1.5824, + "theoretical_loss": 3.4348001775511543, + "tokens_seen": 1969487872 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041012121212121214, + "loss": 1.3634, + "theoretical_loss": 3.4347906192396893, + "tokens_seen": 1969553408 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.27834799885749817, + "objective/train/avg_token_score": 0.007226386573165655, + "objective/train/docs_used": 1113989, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9873694777488708, + "objective/train/original_loss": 4.861571788787842, + "objective/train/theoretical_loss": 3.4347810613353182, + "objective/train/tokens_used": 328478176, + "objective/train/unlikelihood_loss": 0.06607561558485031, + "theoretical_loss": 3.4347810613353182, + "tokens_seen": 1969618944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101010101010101, + "loss": 1.2623, + "theoretical_loss": 3.4347810613353182, + "tokens_seen": 1969618944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100808080808081, + "loss": 1.5619, + "theoretical_loss": 3.434771503838009, + "tokens_seen": 1969684480 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041006060606060607, + "loss": 1.1316, + "theoretical_loss": 3.434761946747732, + "tokens_seen": 1969750016 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410040404040404, + "loss": 1.38, + "theoretical_loss": 3.434752390064456, + "tokens_seen": 1969815552 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041002020202020203, + "loss": 1.4082, + "theoretical_loss": 3.4347428337881496, + "tokens_seen": 1969881088 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041, + "loss": 1.2224, + "theoretical_loss": 3.434733277918782, + "tokens_seen": 1969946624 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409979797979798, + "loss": 1.3039, + "theoretical_loss": 3.4347237224563223, + "tokens_seen": 1970012160 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040995959595959596, + "loss": 1.3388, + "theoretical_loss": 3.4347141674007404, + "tokens_seen": 1970077696 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040993939393939397, + "loss": 1.3502, + "theoretical_loss": 3.4347046127520047, + "tokens_seen": 1970143232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099191919191919, + "loss": 1.3016, + "theoretical_loss": 3.4346950585100853, + "tokens_seen": 1970208768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098989898989899, + "loss": 1.3919, + "theoretical_loss": 3.43468550467495, + "tokens_seen": 1970274304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098787878787879, + "loss": 1.5056, + "theoretical_loss": 3.434675951246568, + "tokens_seen": 1970339840 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040985858585858585, + "loss": 1.2892, + "theoretical_loss": 3.43466639822491, + "tokens_seen": 1970405376 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040983838383838386, + "loss": 1.4575, + "theoretical_loss": 3.434656845609944, + "tokens_seen": 1970470912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098181818181818, + "loss": 1.4339, + "theoretical_loss": 3.434647293401639, + "tokens_seen": 1970536448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097979797979798, + "loss": 1.1932, + "theoretical_loss": 3.434637741599965, + "tokens_seen": 1970601984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097777777777778, + "loss": 1.4734, + "theoretical_loss": 3.4346281902048905, + "tokens_seen": 1970667520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097575757575758, + "loss": 1.4094, + "theoretical_loss": 3.4346186392163847, + "tokens_seen": 1970733056 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040973737373737376, + "loss": 1.5061, + "theoretical_loss": 3.434609088634417, + "tokens_seen": 1970798592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097171717171717, + "loss": 1.7504, + "theoretical_loss": 3.4345995384589565, + "tokens_seen": 1970864128 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040969696969696967, + "loss": 1.6252, + "theoretical_loss": 3.4345899886899725, + "tokens_seen": 1970929664 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096767676767677, + "loss": 1.6023, + "theoretical_loss": 3.434580439327434, + "tokens_seen": 1970995200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096565656565657, + "loss": 1.4679, + "theoretical_loss": 3.4345708903713104, + "tokens_seen": 1971060736 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040963636363636365, + "loss": 1.3235, + "theoretical_loss": 3.4345613418215706, + "tokens_seen": 1971126272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096161616161616, + "loss": 1.1655, + "theoretical_loss": 3.4345517936781835, + "tokens_seen": 1971191808 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.5766128897666931, + "objective/train/avg_token_score": 0.006864483468234539, + "objective/train/docs_used": 1114737, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.9049957990646362, + "objective/train/original_loss": 4.060426235198975, + "objective/train/theoretical_loss": 3.4345422459411195, + "objective/train/tokens_used": 330116576, + "objective/train/unlikelihood_loss": 0.039708707481622696, + "theoretical_loss": 3.4345422459411195, + "tokens_seen": 1971257344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040959595959595956, + "loss": 1.6513, + "theoretical_loss": 3.4345422459411195, + "tokens_seen": 1971257344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040957575757575763, + "loss": 1.5602, + "theoretical_loss": 3.434532698610347, + "tokens_seen": 1971322880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095555555555556, + "loss": 1.3076, + "theoretical_loss": 3.434523151685835, + "tokens_seen": 1971388416 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040953535353535354, + "loss": 1.1315, + "theoretical_loss": 3.4345136051675524, + "tokens_seen": 1971453952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095151515151515, + "loss": 1.3409, + "theoretical_loss": 3.43450405905547, + "tokens_seen": 1971519488 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040949494949494945, + "loss": 1.1566, + "theoretical_loss": 3.4344945133495552, + "tokens_seen": 1971585024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094747474747475, + "loss": 1.4211, + "theoretical_loss": 3.4344849680497784, + "tokens_seen": 1971650560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094545454545455, + "loss": 1.323, + "theoretical_loss": 3.4344754231561083, + "tokens_seen": 1971716096 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040943434343434343, + "loss": 1.5279, + "theoretical_loss": 3.4344658786685143, + "tokens_seen": 1971781632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094141414141414, + "loss": 1.3783, + "theoretical_loss": 3.4344563345869656, + "tokens_seen": 1971847168 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040939393939393946, + "loss": 1.3743, + "theoretical_loss": 3.4344467909114313, + "tokens_seen": 1971912704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093737373737374, + "loss": 1.3636, + "theoretical_loss": 3.4344372476418807, + "tokens_seen": 1971978240 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040935353535353537, + "loss": 1.3911, + "theoretical_loss": 3.434427704778283, + "tokens_seen": 1972043776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093333333333333, + "loss": 1.4582, + "theoretical_loss": 3.434418162320608, + "tokens_seen": 1972109312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093131313131313, + "loss": 1.1357, + "theoretical_loss": 3.434408620268824, + "tokens_seen": 1972174848 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040929292929292935, + "loss": 1.5361, + "theoretical_loss": 3.434399078622901, + "tokens_seen": 1972240384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092727272727273, + "loss": 1.7598, + "theoretical_loss": 3.434389537382808, + "tokens_seen": 1972305920 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040925252525252526, + "loss": 1.365, + "theoretical_loss": 3.434379996548514, + "tokens_seen": 1972371456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092323232323232, + "loss": 1.533, + "theoretical_loss": 3.4343704561199884, + "tokens_seen": 1972436992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092121212121212, + "loss": 1.4524, + "theoretical_loss": 3.4343609160972006, + "tokens_seen": 1972502528 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040919191919191924, + "loss": 1.4198, + "theoretical_loss": 3.4343513764801203, + "tokens_seen": 1972568064 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091717171717172, + "loss": 1.3793, + "theoretical_loss": 3.4343418372687156, + "tokens_seen": 1972633600 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040915151515151515, + "loss": 1.3158, + "theoretical_loss": 3.434332298462957, + "tokens_seen": 1972699136 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091313131313131, + "loss": 1.382, + "theoretical_loss": 3.434322760062813, + "tokens_seen": 1972764672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091111111111111, + "loss": 1.3222, + "theoretical_loss": 3.434313222068253, + "tokens_seen": 1972830208 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.38007086515426636, + "objective/train/avg_token_score": 0.014573216438293457, + "objective/train/docs_used": 1116152, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2298966646194458, + "objective/train/original_loss": 4.61524772644043, + "objective/train/theoretical_loss": 3.4343036844792465, + "objective/train/tokens_used": 331754976, + "objective/train/unlikelihood_loss": 0.04789040982723236, + "theoretical_loss": 3.4343036844792465, + "tokens_seen": 1972895744 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040909090909090913, + "loss": 1.2967, + "theoretical_loss": 3.4343036844792465, + "tokens_seen": 1972895744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090707070707071, + "loss": 1.4073, + "theoretical_loss": 3.4342941472957627, + "tokens_seen": 1972961280 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040905050505050505, + "loss": 1.4403, + "theoretical_loss": 3.4342846105177705, + "tokens_seen": 1973026816 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409030303030303, + "loss": 1.3333, + "theoretical_loss": 3.4342750741452397, + "tokens_seen": 1973092352 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409010101010101, + "loss": 1.5288, + "theoretical_loss": 3.43426553817814, + "tokens_seen": 1973157888 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408989898989899, + "loss": 1.3896, + "theoretical_loss": 3.4342560026164395, + "tokens_seen": 1973223424 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408969696969697, + "loss": 1.3534, + "theoretical_loss": 3.4342464674601083, + "tokens_seen": 1973288960 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040894949494949494, + "loss": 1.3614, + "theoretical_loss": 3.4342369327091156, + "tokens_seen": 1973354496 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040892929292929295, + "loss": 1.6497, + "theoretical_loss": 3.434227398363431, + "tokens_seen": 1973420032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089090909090909, + "loss": 1.577, + "theoretical_loss": 3.4342178644230232, + "tokens_seen": 1973485568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088888888888889, + "loss": 1.2795, + "theoretical_loss": 3.4342083308878615, + "tokens_seen": 1973551104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088686868686869, + "loss": 1.4452, + "theoretical_loss": 3.4341987977579156, + "tokens_seen": 1973616640 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040884848484848483, + "loss": 1.4619, + "theoretical_loss": 3.434189265033155, + "tokens_seen": 1973682176 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040882828282828284, + "loss": 1.4424, + "theoretical_loss": 3.4341797327135484, + "tokens_seen": 1973747712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088080808080808, + "loss": 1.1546, + "theoretical_loss": 3.4341702007990658, + "tokens_seen": 1973813248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087878787878788, + "loss": 1.4294, + "theoretical_loss": 3.434160669289676, + "tokens_seen": 1973878784 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040876767676767677, + "loss": 0.9521, + "theoretical_loss": 3.4341511381853485, + "tokens_seen": 1973944320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087474747474748, + "loss": 1.5766, + "theoretical_loss": 3.434141607486053, + "tokens_seen": 1974009856 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040872727272727274, + "loss": 1.4657, + "theoretical_loss": 3.4341320771917583, + "tokens_seen": 1974075392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087070707070707, + "loss": 1.4426, + "theoretical_loss": 3.434122547302434, + "tokens_seen": 1974140928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086868686868687, + "loss": 1.5492, + "theoretical_loss": 3.4341130178180492, + "tokens_seen": 1974206464 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040866666666666666, + "loss": 1.3706, + "theoretical_loss": 3.4341034887385735, + "tokens_seen": 1974272000 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040864646464646467, + "loss": 1.5491, + "theoretical_loss": 3.4340939600639766, + "tokens_seen": 1974337536 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040862626262626263, + "loss": 1.3823, + "theoretical_loss": 3.4340844317942274, + "tokens_seen": 1974403072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086060606060606, + "loss": 1.3105, + "theoretical_loss": 3.4340749039292957, + "tokens_seen": 1974468608 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.46144917607307434, + "objective/train/avg_token_score": 0.03224779665470123, + "objective/train/docs_used": 1116806, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.653957486152649, + "objective/train/original_loss": 4.292056560516357, + "objective/train/theoretical_loss": 3.4340653764691496, + "objective/train/tokens_used": 333393376, + "objective/train/unlikelihood_loss": 0.05980272591114044, + "theoretical_loss": 3.4340653764691496, + "tokens_seen": 1974534144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085858585858586, + "loss": 1.4107, + "theoretical_loss": 3.4340653764691496, + "tokens_seen": 1974534144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085656565656566, + "loss": 1.4226, + "theoretical_loss": 3.43405584941376, + "tokens_seen": 1974599680 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040854545454545457, + "loss": 1.3297, + "theoretical_loss": 3.4340463227630957, + "tokens_seen": 1974665216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085252525252525, + "loss": 1.4029, + "theoretical_loss": 3.434036796517126, + "tokens_seen": 1974730752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085050505050505, + "loss": 1.3152, + "theoretical_loss": 3.43402727067582, + "tokens_seen": 1974796288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084848484848485, + "loss": 1.6887, + "theoretical_loss": 3.4340177452391476, + "tokens_seen": 1974861824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084646464646465, + "loss": 1.3779, + "theoretical_loss": 3.4340082202070774, + "tokens_seen": 1974927360 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040844444444444446, + "loss": 1.3655, + "theoretical_loss": 3.43399869557958, + "tokens_seen": 1974992896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084242424242424, + "loss": 1.5568, + "theoretical_loss": 3.4339891713566244, + "tokens_seen": 1975058432 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040840404040404037, + "loss": 1.4729, + "theoretical_loss": 3.433979647538179, + "tokens_seen": 1975123968 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040838383838383844, + "loss": 1.3622, + "theoretical_loss": 3.4339701241242144, + "tokens_seen": 1975189504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083636363636364, + "loss": 1.2509, + "theoretical_loss": 3.4339606011146993, + "tokens_seen": 1975255040 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040834343434343435, + "loss": 1.2185, + "theoretical_loss": 3.4339510785096037, + "tokens_seen": 1975320576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083232323232323, + "loss": 1.4309, + "theoretical_loss": 3.4339415563088966, + "tokens_seen": 1975386112 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040830303030303026, + "loss": 1.4844, + "theoretical_loss": 3.433932034512547, + "tokens_seen": 1975451648 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040828282828282833, + "loss": 1.2712, + "theoretical_loss": 3.4339225131205255, + "tokens_seen": 1975517184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082626262626263, + "loss": 1.4414, + "theoretical_loss": 3.4339129921328, + "tokens_seen": 1975582720 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040824242424242424, + "loss": 1.5263, + "theoretical_loss": 3.4339034715493417, + "tokens_seen": 1975648256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082222222222222, + "loss": 1.1537, + "theoretical_loss": 3.433893951370118, + "tokens_seen": 1975713792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082020202020202, + "loss": 1.4704, + "theoretical_loss": 3.4338844315951, + "tokens_seen": 1975779328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081818181818182, + "loss": 1.41, + "theoretical_loss": 3.4338749122242564, + "tokens_seen": 1975844864 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081616161616162, + "loss": 1.5023, + "theoretical_loss": 3.433865393257557, + "tokens_seen": 1975910400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040814141414141414, + "loss": 1.5095, + "theoretical_loss": 3.433855874694971, + "tokens_seen": 1975975936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081212121212121, + "loss": 1.5097, + "theoretical_loss": 3.433846356536468, + "tokens_seen": 1976041472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081010101010101, + "loss": 1.1897, + "theoretical_loss": 3.4338368387820166, + "tokens_seen": 1976107008 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.38624143600463867, + "objective/train/avg_token_score": 0.0034812039230018854, + "objective/train/docs_used": 1117380, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2491487264633179, + "objective/train/original_loss": 4.108737945556641, + "objective/train/theoretical_loss": 3.433827321431587, + "objective/train/tokens_used": 335031776, + "objective/train/unlikelihood_loss": 0.0697765201330185, + "theoretical_loss": 3.433827321431587, + "tokens_seen": 1976172544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004080808080808081, + "loss": 1.4446, + "theoretical_loss": 3.433827321431587, + "tokens_seen": 1976172544 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040806060606060607, + "loss": 1.293, + "theoretical_loss": 3.433817804485149, + "tokens_seen": 1976238080 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040804040404040403, + "loss": 1.415, + "theoretical_loss": 3.4338082879426715, + "tokens_seen": 1976303616 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040802020202020204, + "loss": 1.3463, + "theoretical_loss": 3.433798771804124, + "tokens_seen": 1976369152 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408, + "loss": 1.6793, + "theoretical_loss": 3.433789256069476, + "tokens_seen": 1976434688 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407979797979798, + "loss": 1.3098, + "theoretical_loss": 3.433779740738697, + "tokens_seen": 1976500224 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040795959595959596, + "loss": 1.4687, + "theoretical_loss": 3.433770225811757, + "tokens_seen": 1976565760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079393939393939, + "loss": 1.6195, + "theoretical_loss": 3.433760711288624, + "tokens_seen": 1976631296 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040791919191919193, + "loss": 1.4593, + "theoretical_loss": 3.4337511971692694, + "tokens_seen": 1976696832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078989898989899, + "loss": 1.4784, + "theoretical_loss": 3.433741683453661, + "tokens_seen": 1976762368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078787878787879, + "loss": 1.4241, + "theoretical_loss": 3.4337321701417696, + "tokens_seen": 1976827904 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040785858585858586, + "loss": 1.6483, + "theoretical_loss": 3.433722657233564, + "tokens_seen": 1976893440 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040783838383838387, + "loss": 1.3813, + "theoretical_loss": 3.4337131447290132, + "tokens_seen": 1976958976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078181818181818, + "loss": 1.5208, + "theoretical_loss": 3.4337036326280876, + "tokens_seen": 1977024512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077979797979798, + "loss": 1.3666, + "theoretical_loss": 3.433694120930756, + "tokens_seen": 1977090048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077777777777778, + "loss": 1.5639, + "theoretical_loss": 3.4336846096369893, + "tokens_seen": 1977155584 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040775757575757575, + "loss": 1.3248, + "theoretical_loss": 3.433675098746755, + "tokens_seen": 1977221120 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040773737373737376, + "loss": 1.2001, + "theoretical_loss": 3.4336655882600238, + "tokens_seen": 1977286656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077171717171717, + "loss": 1.3965, + "theoretical_loss": 3.433656078176765, + "tokens_seen": 1977352192 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040769696969696973, + "loss": 1.3558, + "theoretical_loss": 3.4336465684969477, + "tokens_seen": 1977417728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076767676767677, + "loss": 1.4859, + "theoretical_loss": 3.4336370592205423, + "tokens_seen": 1977483264 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076565656565657, + "loss": 1.3785, + "theoretical_loss": 3.4336275503475178, + "tokens_seen": 1977548800 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040763636363636365, + "loss": 1.6469, + "theoretical_loss": 3.4336180418778435, + "tokens_seen": 1977614336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076161616161616, + "loss": 1.3878, + "theoretical_loss": 3.4336085338114897, + "tokens_seen": 1977679872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075959595959596, + "loss": 1.2246, + "theoretical_loss": 3.433599026148425, + "tokens_seen": 1977745408 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.2834188640117645, + "objective/train/avg_token_score": 0.007966935634613037, + "objective/train/docs_used": 1118619, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.9379705190658569, + "objective/train/original_loss": 4.463184356689453, + "objective/train/theoretical_loss": 3.4335895188886187, + "objective/train/tokens_used": 336670176, + "objective/train/unlikelihood_loss": 0.07986027747392654, + "theoretical_loss": 3.4335895188886187, + "tokens_seen": 1977810944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075757575757576, + "loss": 1.4292, + "theoretical_loss": 3.4335895188886187, + "tokens_seen": 1977810944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075555555555556, + "loss": 1.4095, + "theoretical_loss": 3.433580012032042, + "tokens_seen": 1977876480 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040753535353535355, + "loss": 1.4814, + "theoretical_loss": 3.433570505578663, + "tokens_seen": 1977942016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075151515151515, + "loss": 1.5789, + "theoretical_loss": 3.433560999528451, + "tokens_seen": 1978007552 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074949494949495, + "loss": 1.5362, + "theoretical_loss": 3.433551493881377, + "tokens_seen": 1978073088 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040747474747474747, + "loss": 1.3804, + "theoretical_loss": 3.4335419886374092, + "tokens_seen": 1978138624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074545454545455, + "loss": 1.6188, + "theoretical_loss": 3.433532483796518, + "tokens_seen": 1978204160 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040743434343434344, + "loss": 1.1727, + "theoretical_loss": 3.433522979358673, + "tokens_seen": 1978269696 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074141414141414, + "loss": 1.5376, + "theoretical_loss": 3.433513475323843, + "tokens_seen": 1978335232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073939393939394, + "loss": 1.2815, + "theoretical_loss": 3.433503971691998, + "tokens_seen": 1978400768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073737373737374, + "loss": 1.4291, + "theoretical_loss": 3.433494468463107, + "tokens_seen": 1978466304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073535353535354, + "loss": 1.3798, + "theoretical_loss": 3.4334849656371405, + "tokens_seen": 1978531840 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040733333333333333, + "loss": 1.3716, + "theoretical_loss": 3.4334754632140676, + "tokens_seen": 1978597376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073131313131313, + "loss": 1.327, + "theoretical_loss": 3.433465961193858, + "tokens_seen": 1978662912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072929292929293, + "loss": 1.3915, + "theoretical_loss": 3.4334564595764814, + "tokens_seen": 1978728448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072727272727273, + "loss": 1.543, + "theoretical_loss": 3.4334469583619067, + "tokens_seen": 1978793984 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040725252525252527, + "loss": 1.3711, + "theoretical_loss": 3.433437457550104, + "tokens_seen": 1978859520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072323232323232, + "loss": 1.2373, + "theoretical_loss": 3.4334279571410438, + "tokens_seen": 1978925056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072121212121212, + "loss": 1.4841, + "theoretical_loss": 3.4334184571346933, + "tokens_seen": 1978990592 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040719191919191925, + "loss": 1.5336, + "theoretical_loss": 3.4334089575310247, + "tokens_seen": 1979056128 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071717171717172, + "loss": 1.4586, + "theoretical_loss": 3.433399458330006, + "tokens_seen": 1979121664 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040715151515151516, + "loss": 1.2878, + "theoretical_loss": 3.433389959531607, + "tokens_seen": 1979187200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071313131313131, + "loss": 1.5874, + "theoretical_loss": 3.433380461135798, + "tokens_seen": 1979252736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071111111111111, + "loss": 1.5125, + "theoretical_loss": 3.433370963142548, + "tokens_seen": 1979318272 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040709090909090914, + "loss": 1.7091, + "theoretical_loss": 3.433361465551827, + "tokens_seen": 1979383808 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.34341397881507874, + "objective/train/avg_token_score": 0.009266973473131657, + "objective/train/docs_used": 1119354, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.2784351110458374, + "objective/train/original_loss": 4.967345714569092, + "objective/train/theoretical_loss": 3.4333519683636036, + "objective/train/tokens_used": 338308576, + "objective/train/unlikelihood_loss": 0.047723278403282166, + "theoretical_loss": 3.4333519683636036, + "tokens_seen": 1979449344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070707070707071, + "loss": 1.5983, + "theoretical_loss": 3.4333519683636036, + "tokens_seen": 1979449344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040705050505050505, + "loss": 1.3467, + "theoretical_loss": 3.433342471577849, + "tokens_seen": 1979514880 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407030303030303, + "loss": 1.4703, + "theoretical_loss": 3.4333329751945314, + "tokens_seen": 1979580416 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407010101010101, + "loss": 1.607, + "theoretical_loss": 3.433323479213622, + "tokens_seen": 1979645952 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040698989898989903, + "loss": 1.3784, + "theoretical_loss": 3.4333139836350886, + "tokens_seen": 1979711488 + }, + { + "epoch": 0.19, + "learning_rate": 0.000406969696969697, + "loss": 1.3995, + "theoretical_loss": 3.433304488458902, + "tokens_seen": 1979777024 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040694949494949495, + "loss": 1.6541, + "theoretical_loss": 3.433294993685031, + "tokens_seen": 1979842560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069292929292929, + "loss": 1.2676, + "theoretical_loss": 3.4332854993134463, + "tokens_seen": 1979908096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069090909090909, + "loss": 1.3127, + "theoretical_loss": 3.4332760053441174, + "tokens_seen": 1979973632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068888888888889, + "loss": 1.4902, + "theoretical_loss": 3.4332665117770125, + "tokens_seen": 1980039168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068686868686869, + "loss": 1.1425, + "theoretical_loss": 3.433257018612103, + "tokens_seen": 1980104704 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040684848484848484, + "loss": 1.5914, + "theoretical_loss": 3.433247525849358, + "tokens_seen": 1980170240 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040682828282828285, + "loss": 1.4876, + "theoretical_loss": 3.433238033488747, + "tokens_seen": 1980235776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068080808080808, + "loss": 1.5058, + "theoretical_loss": 3.433228541530239, + "tokens_seen": 1980301312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067878787878788, + "loss": 1.3863, + "theoretical_loss": 3.433219049973805, + "tokens_seen": 1980366848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067676767676768, + "loss": 1.5363, + "theoretical_loss": 3.4332095588194136, + "tokens_seen": 1980432384 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040674747474747473, + "loss": 1.1681, + "theoretical_loss": 3.433200068067035, + "tokens_seen": 1980497920 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040672727272727274, + "loss": 1.3775, + "theoretical_loss": 3.4331905777166387, + "tokens_seen": 1980563456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067070707070707, + "loss": 1.5538, + "theoretical_loss": 3.4331810877681948, + "tokens_seen": 1980628992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066868686868687, + "loss": 1.3664, + "theoretical_loss": 3.433171598221672, + "tokens_seen": 1980694528 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040666666666666667, + "loss": 1.2929, + "theoretical_loss": 3.43316210907704, + "tokens_seen": 1980760064 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066464646464647, + "loss": 1.6327, + "theoretical_loss": 3.43315262033427, + "tokens_seen": 1980825600 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040662626262626264, + "loss": 1.3788, + "theoretical_loss": 3.4331431319933303, + "tokens_seen": 1980891136 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066060606060606, + "loss": 1.3966, + "theoretical_loss": 3.4331336440541915, + "tokens_seen": 1980956672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004065858585858586, + "loss": 1.5779, + "theoretical_loss": 3.433124156516823, + "tokens_seen": 1981022208 + }, + { + "epoch": 0.19, + "objective/train/aligned_token_count": 0.41037389636039734, + "objective/train/avg_token_score": 0.023919859901070595, + "objective/train/docs_used": 1120511, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.3339953422546387, + "objective/train/original_loss": 4.790691375732422, + "objective/train/theoretical_loss": 3.4331146693811934, + "objective/train/tokens_used": 339946976, + "objective/train/unlikelihood_loss": 0.049351904541254044, + "theoretical_loss": 3.4331146693811934, + "tokens_seen": 1981087744 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040656565656565656, + "loss": 1.422, + "theoretical_loss": 3.4331146693811934, + "tokens_seen": 1981087744 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040654545454545457, + "loss": 1.6359, + "theoretical_loss": 3.4331051826472736, + "tokens_seen": 1981153280 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040652525252525253, + "loss": 1.3955, + "theoretical_loss": 3.433095696315033, + "tokens_seen": 1981218816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004065050505050505, + "loss": 1.5511, + "theoretical_loss": 3.4330862103844417, + "tokens_seen": 1981284352 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064848484848485, + "loss": 1.4746, + "theoretical_loss": 3.4330767248554688, + "tokens_seen": 1981349888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064646464646465, + "loss": 1.6425, + "theoretical_loss": 3.433067239728084, + "tokens_seen": 1981415424 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040644444444444446, + "loss": 1.1774, + "theoretical_loss": 3.4330577550022574, + "tokens_seen": 1981480960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064242424242424, + "loss": 1.5055, + "theoretical_loss": 3.4330482706779586, + "tokens_seen": 1981546496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064040404040404, + "loss": 1.3855, + "theoretical_loss": 3.4330387867551573, + "tokens_seen": 1981612032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063838383838384, + "loss": 1.3515, + "theoretical_loss": 3.4330293032338233, + "tokens_seen": 1981677568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063636363636364, + "loss": 1.4944, + "theoretical_loss": 3.433019820113926, + "tokens_seen": 1981743104 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040634343434343436, + "loss": 1.4746, + "theoretical_loss": 3.4330103373954355, + "tokens_seen": 1981808640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063232323232323, + "loss": 1.2959, + "theoretical_loss": 3.4330008550783213, + "tokens_seen": 1981874176 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040630303030303027, + "loss": 1.5334, + "theoretical_loss": 3.4329913731625536, + "tokens_seen": 1981939712 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040628282828282834, + "loss": 1.2631, + "theoretical_loss": 3.4329818916481014, + "tokens_seen": 1982005248 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062626262626263, + "loss": 1.6203, + "theoretical_loss": 3.4329724105349353, + "tokens_seen": 1982070784 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040624242424242425, + "loss": 1.4313, + "theoretical_loss": 3.432962929823024, + "tokens_seen": 1982136320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062222222222222, + "loss": 1.415, + "theoretical_loss": 3.432953449512338, + "tokens_seen": 1982201856 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040620202020202016, + "loss": 1.4721, + "theoretical_loss": 3.432943969602847, + "tokens_seen": 1982267392 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040618181818181823, + "loss": 1.4335, + "theoretical_loss": 3.432934490094521, + "tokens_seen": 1982332928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061616161616162, + "loss": 1.4509, + "theoretical_loss": 3.4329250109873293, + "tokens_seen": 1982398464 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040614141414141414, + "loss": 1.4384, + "theoretical_loss": 3.4329155322812417, + "tokens_seen": 1982464000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061212121212121, + "loss": 1.4729, + "theoretical_loss": 3.432906053976228, + "tokens_seen": 1982529536 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040610101010101006, + "loss": 1.4978, + "theoretical_loss": 3.432896576072258, + "tokens_seen": 1982595072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060808080808081, + "loss": 1.5901, + "theoretical_loss": 3.432887098569301, + "tokens_seen": 1982660608 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.5727028250694275, + "objective/train/avg_token_score": 0.004548849072307348, + "objective/train/docs_used": 1121670, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 2.048727512359619, + "objective/train/original_loss": 4.066925525665283, + "objective/train/theoretical_loss": 3.4328776214673278, + "objective/train/tokens_used": 341585376, + "objective/train/unlikelihood_loss": 0.07542819529771805, + "theoretical_loss": 3.4328776214673278, + "tokens_seen": 1982726144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060606060606061, + "loss": 1.7648, + "theoretical_loss": 3.4328776214673278, + "tokens_seen": 1982726144 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040604040404040403, + "loss": 1.4492, + "theoretical_loss": 3.432868144766308, + "tokens_seen": 1982791680 + }, + { + "epoch": 0.2, + "learning_rate": 0.000406020202020202, + "loss": 1.4899, + "theoretical_loss": 3.432858668466211, + "tokens_seen": 1982857216 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040600000000000006, + "loss": 1.5625, + "theoretical_loss": 3.432849192567006, + "tokens_seen": 1982922752 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405979797979798, + "loss": 1.49, + "theoretical_loss": 3.4328397170686635, + "tokens_seen": 1982988288 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040595959595959597, + "loss": 1.6991, + "theoretical_loss": 3.4328302419711534, + "tokens_seen": 1983053824 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040593939393939393, + "loss": 1.5363, + "theoretical_loss": 3.4328207672744453, + "tokens_seen": 1983119360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004059191919191919, + "loss": 1.3888, + "theoretical_loss": 3.432811292978509, + "tokens_seen": 1983184896 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040589898989898995, + "loss": 1.4623, + "theoretical_loss": 3.432801819083315, + "tokens_seen": 1983250432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058787878787879, + "loss": 1.2032, + "theoretical_loss": 3.4327923455888314, + "tokens_seen": 1983315968 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040585858585858586, + "loss": 1.5196, + "theoretical_loss": 3.4327828724950296, + "tokens_seen": 1983381504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058383838383838, + "loss": 1.3848, + "theoretical_loss": 3.432773399801879, + "tokens_seen": 1983447040 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040581818181818183, + "loss": 1.2752, + "theoretical_loss": 3.432763927509349, + "tokens_seen": 1983512576 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040579797979797984, + "loss": 1.4994, + "theoretical_loss": 3.4327544556174097, + "tokens_seen": 1983578112 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057777777777778, + "loss": 1.5548, + "theoretical_loss": 3.432744984126031, + "tokens_seen": 1983643648 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040575757575757576, + "loss": 1.3647, + "theoretical_loss": 3.4327355130351824, + "tokens_seen": 1983709184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057373737373737, + "loss": 1.3819, + "theoretical_loss": 3.4327260423448345, + "tokens_seen": 1983774720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057171717171717, + "loss": 1.1727, + "theoretical_loss": 3.4327165720549564, + "tokens_seen": 1983840256 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040569696969696973, + "loss": 1.442, + "theoretical_loss": 3.4327071021655176, + "tokens_seen": 1983905792 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056767676767677, + "loss": 1.3211, + "theoretical_loss": 3.432697632676489, + "tokens_seen": 1983971328 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040565656565656565, + "loss": 1.4399, + "theoretical_loss": 3.43268816358784, + "tokens_seen": 1984036864 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040563636363636366, + "loss": 1.401, + "theoretical_loss": 3.4326786948995407, + "tokens_seen": 1984102400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056161616161616, + "loss": 1.7314, + "theoretical_loss": 3.43266922661156, + "tokens_seen": 1984167936 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040559595959595963, + "loss": 1.4885, + "theoretical_loss": 3.4326597587238687, + "tokens_seen": 1984233472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055757575757576, + "loss": 1.193, + "theoretical_loss": 3.432650291236436, + "tokens_seen": 1984299008 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.46981915831565857, + "objective/train/avg_token_score": 0.007897254079580307, + "objective/train/docs_used": 1122413, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.6054514646530151, + "objective/train/original_loss": 4.072227954864502, + "objective/train/theoretical_loss": 3.4326408241492326, + "objective/train/tokens_used": 343223776, + "objective/train/unlikelihood_loss": 0.05646788701415062, + "theoretical_loss": 3.4326408241492326, + "tokens_seen": 1984364544 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040555555555555554, + "loss": 1.4734, + "theoretical_loss": 3.4326408241492326, + "tokens_seen": 1984364544 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040553535353535355, + "loss": 1.5419, + "theoretical_loss": 3.4326313574622276, + "tokens_seen": 1984430080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055151515151515, + "loss": 1.4651, + "theoretical_loss": 3.432621891175391, + "tokens_seen": 1984495616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054949494949495, + "loss": 1.5656, + "theoretical_loss": 3.432612425288693, + "tokens_seen": 1984561152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054747474747475, + "loss": 1.3567, + "theoretical_loss": 3.4326029598021033, + "tokens_seen": 1984626688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054545454545455, + "loss": 1.5375, + "theoretical_loss": 3.432593494715592, + "tokens_seen": 1984692224 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040543434343434344, + "loss": 1.2787, + "theoretical_loss": 3.432584030029128, + "tokens_seen": 1984757760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054141414141414, + "loss": 1.5412, + "theoretical_loss": 3.4325745657426827, + "tokens_seen": 1984823296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053939393939394, + "loss": 1.6001, + "theoretical_loss": 3.432565101856224, + "tokens_seen": 1984888832 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040537373737373737, + "loss": 1.4601, + "theoretical_loss": 3.4325556383697244, + "tokens_seen": 1984954368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053535353535354, + "loss": 1.4659, + "theoretical_loss": 3.432546175283152, + "tokens_seen": 1985019904 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040533333333333334, + "loss": 1.4451, + "theoretical_loss": 3.4325367125964767, + "tokens_seen": 1985085440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053131313131313, + "loss": 1.476, + "theoretical_loss": 3.4325272503096693, + "tokens_seen": 1985150976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052929292929293, + "loss": 1.4559, + "theoretical_loss": 3.4325177884226985, + "tokens_seen": 1985216512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052727272727273, + "loss": 1.4112, + "theoretical_loss": 3.4325083269355354, + "tokens_seen": 1985282048 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052525252525253, + "loss": 1.6486, + "theoretical_loss": 3.4324988658481494, + "tokens_seen": 1985347584 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040523232323232323, + "loss": 1.5827, + "theoretical_loss": 3.43248940516051, + "tokens_seen": 1985413120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052121212121212, + "loss": 1.5105, + "theoretical_loss": 3.432479944872588, + "tokens_seen": 1985478656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051919191919192, + "loss": 1.4755, + "theoretical_loss": 3.4324704849843526, + "tokens_seen": 1985544192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051717171717172, + "loss": 1.5322, + "theoretical_loss": 3.4324610254957744, + "tokens_seen": 1985609728 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040515151515151517, + "loss": 1.359, + "theoretical_loss": 3.4324515664068223, + "tokens_seen": 1985675264 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051313131313131, + "loss": 1.3518, + "theoretical_loss": 3.432442107717467, + "tokens_seen": 1985740800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051111111111111, + "loss": 1.5694, + "theoretical_loss": 3.4324326494276782, + "tokens_seen": 1985806336 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040509090909090915, + "loss": 1.2736, + "theoretical_loss": 3.432423191537426, + "tokens_seen": 1985871872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050707070707071, + "loss": 1.4792, + "theoretical_loss": 3.4324137340466807, + "tokens_seen": 1985937408 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.2547653913497925, + "objective/train/avg_token_score": 0.016683774068951607, + "objective/train/docs_used": 1123893, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.856845498085022, + "objective/train/original_loss": 5.153318881988525, + "objective/train/theoretical_loss": 3.432404276955411, + "objective/train/tokens_used": 344862176, + "objective/train/unlikelihood_loss": 0.05154005065560341, + "theoretical_loss": 3.432404276955411, + "tokens_seen": 1986002944 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040505050505050506, + "loss": 1.2664, + "theoretical_loss": 3.432404276955411, + "tokens_seen": 1986002944 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405030303030303, + "loss": 1.6352, + "theoretical_loss": 3.432394820263588, + "tokens_seen": 1986068480 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040501010101010097, + "loss": 1.4826, + "theoretical_loss": 3.432385363971181, + "tokens_seen": 1986134016 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040498989898989904, + "loss": 1.3789, + "theoretical_loss": 3.4323759080781606, + "tokens_seen": 1986199552 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404969696969697, + "loss": 1.4896, + "theoretical_loss": 3.432366452584496, + "tokens_seen": 1986265088 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040494949494949495, + "loss": 1.5823, + "theoretical_loss": 3.4323569974901575, + "tokens_seen": 1986330624 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049292929292929, + "loss": 1.5142, + "theoretical_loss": 3.4323475427951156, + "tokens_seen": 1986396160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049090909090909, + "loss": 1.4073, + "theoretical_loss": 3.4323380884993395, + "tokens_seen": 1986461696 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040488888888888893, + "loss": 1.4661, + "theoretical_loss": 3.4323286346027997, + "tokens_seen": 1986527232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048686868686869, + "loss": 1.5341, + "theoretical_loss": 3.4323191811054654, + "tokens_seen": 1986592768 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040484848484848484, + "loss": 1.314, + "theoretical_loss": 3.432309728007307, + "tokens_seen": 1986658304 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048282828282828, + "loss": 1.2981, + "theoretical_loss": 3.432300275308295, + "tokens_seen": 1986723840 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048080808080808, + "loss": 1.372, + "theoretical_loss": 3.4322908230083984, + "tokens_seen": 1986789376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047878787878788, + "loss": 1.4414, + "theoretical_loss": 3.432281371107588, + "tokens_seen": 1986854912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047676767676768, + "loss": 1.6455, + "theoretical_loss": 3.4322719196058338, + "tokens_seen": 1986920448 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040474747474747474, + "loss": 1.3235, + "theoretical_loss": 3.4322624685031053, + "tokens_seen": 1986985984 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040472727272727275, + "loss": 1.3656, + "theoretical_loss": 3.4322530177993724, + "tokens_seen": 1987051520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047070707070707, + "loss": 1.3143, + "theoretical_loss": 3.4322435674946057, + "tokens_seen": 1987117056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046868686868687, + "loss": 1.6096, + "theoretical_loss": 3.4322341175887745, + "tokens_seen": 1987182592 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040466666666666667, + "loss": 1.2566, + "theoretical_loss": 3.4322246680818496, + "tokens_seen": 1987248128 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040464646464646463, + "loss": 1.2232, + "theoretical_loss": 3.4322152189738, + "tokens_seen": 1987313664 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040462626262626264, + "loss": 1.5018, + "theoretical_loss": 3.4322057702645967, + "tokens_seen": 1987379200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046060606060606, + "loss": 1.5121, + "theoretical_loss": 3.4321963219542098, + "tokens_seen": 1987444736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045858585858586, + "loss": 1.3607, + "theoretical_loss": 3.4321868740426082, + "tokens_seen": 1987510272 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040456565656565657, + "loss": 1.818, + "theoretical_loss": 3.432177426529763, + "tokens_seen": 1987575808 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.5406891703605652, + "objective/train/avg_token_score": 0.005704171024262905, + "objective/train/docs_used": 1124511, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.8316115140914917, + "objective/train/original_loss": 3.5460124015808105, + "objective/train/theoretical_loss": 3.4321679794156434, + "objective/train/tokens_used": 346500576, + "objective/train/unlikelihood_loss": 0.07432541251182556, + "theoretical_loss": 3.4321679794156434, + "tokens_seen": 1987641344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045454545454546, + "loss": 1.5832, + "theoretical_loss": 3.4321679794156434, + "tokens_seen": 1987641344 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040452525252525253, + "loss": 1.3207, + "theoretical_loss": 3.4321585327002198, + "tokens_seen": 1987706880 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045050505050505, + "loss": 1.3359, + "theoretical_loss": 3.432149086383462, + "tokens_seen": 1987772416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044848484848485, + "loss": 1.4143, + "theoretical_loss": 3.4321396404653406, + "tokens_seen": 1987837952 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040446464646464646, + "loss": 1.3399, + "theoretical_loss": 3.4321301949458256, + "tokens_seen": 1987903488 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040444444444444447, + "loss": 1.4967, + "theoretical_loss": 3.432120749824886, + "tokens_seen": 1987969024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044242424242424, + "loss": 1.6424, + "theoretical_loss": 3.4321113051024934, + "tokens_seen": 1988034560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044040404040404, + "loss": 1.656, + "theoretical_loss": 3.4321018607786167, + "tokens_seen": 1988100096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004043838383838384, + "loss": 1.3635, + "theoretical_loss": 3.4320924168532256, + "tokens_seen": 1988165632 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040436363636363635, + "loss": 1.4125, + "theoretical_loss": 3.432082973326292, + "tokens_seen": 1988231168 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040434343434343436, + "loss": 1.5523, + "theoretical_loss": 3.432073530197784, + "tokens_seen": 1988296704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004043232323232323, + "loss": 1.4857, + "theoretical_loss": 3.4320640874676727, + "tokens_seen": 1988362240 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040430303030303033, + "loss": 1.3104, + "theoretical_loss": 3.4320546451359277, + "tokens_seen": 1988427776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042828282828283, + "loss": 1.4525, + "theoretical_loss": 3.4320452032025193, + "tokens_seen": 1988493312 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042626262626263, + "loss": 1.4042, + "theoretical_loss": 3.4320357616674175, + "tokens_seen": 1988558848 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040424242424242425, + "loss": 1.387, + "theoretical_loss": 3.4320263205305928, + "tokens_seen": 1988624384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042222222222222, + "loss": 1.398, + "theoretical_loss": 3.4320168797920143, + "tokens_seen": 1988689920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042020202020202, + "loss": 1.3234, + "theoretical_loss": 3.432007439451653, + "tokens_seen": 1988755456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041818181818182, + "loss": 1.6412, + "theoretical_loss": 3.4319979995094787, + "tokens_seen": 1988820992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041616161616162, + "loss": 1.5261, + "theoretical_loss": 3.431988559965461, + "tokens_seen": 1988886528 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040414141414141415, + "loss": 1.5544, + "theoretical_loss": 3.431979120819571, + "tokens_seen": 1988952064 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041212121212121, + "loss": 1.4711, + "theoretical_loss": 3.4319696820717778, + "tokens_seen": 1989017600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041010101010101, + "loss": 1.2866, + "theoretical_loss": 3.431960243722052, + "tokens_seen": 1989083136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040808080808081, + "loss": 1.435, + "theoretical_loss": 3.4319508057703634, + "tokens_seen": 1989148672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040606060606061, + "loss": 1.4636, + "theoretical_loss": 3.4319413682166826, + "tokens_seen": 1989214208 + }, + { + "debugging/Self-BLEU-5": 0.4683363022964717, + "debugging/distinct-1-grams": 0.8034092501484725, + "debugging/distinct-2-grams": 0.9641002767459668, + "debugging/entropy-1-grams": 6.198537273223117, + "debugging/entropy-2-grams": 7.019141508423205, + "debugging/length": 468.3529411764706, + "debugging/num_segments": 17, + "epoch": 0.2, + "objective/train/aligned_token_count": 0.21560361981391907, + "objective/train/avg_token_score": 0.04828590154647827, + "objective/train/docs_used": 1125297, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8591703772544861, + "objective/train/original_loss": 5.175046443939209, + "objective/train/theoretical_loss": 3.431931931060979, + "objective/train/tokens_used": 348138976, + "objective/train/unlikelihood_loss": 0.05250702425837517, + "theoretical_loss": 3.431931931060979, + "tokens_seen": 1989279744 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040404040404040404, + "loss": 1.3473, + "theoretical_loss": 3.431931931060979, + "tokens_seen": 1989279744 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404020202020202, + "loss": 1.4626, + "theoretical_loss": 3.4319224943032234, + "tokens_seen": 1989345280 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404, + "loss": 1.278, + "theoretical_loss": 3.431913057943386, + "tokens_seen": 1989410816 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403979797979798, + "loss": 1.5107, + "theoretical_loss": 3.4319036219814363, + "tokens_seen": 1989476352 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403959595959596, + "loss": 1.5861, + "theoretical_loss": 3.4318941864173445, + "tokens_seen": 1989541888 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040393939393939393, + "loss": 1.2866, + "theoretical_loss": 3.4318847512510806, + "tokens_seen": 1989607424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039191919191919, + "loss": 1.5729, + "theoretical_loss": 3.431875316482615, + "tokens_seen": 1989672960 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040389898989898995, + "loss": 1.3685, + "theoretical_loss": 3.4318658821119183, + "tokens_seen": 1989738496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004038787878787879, + "loss": 1.5106, + "theoretical_loss": 3.43185644813896, + "tokens_seen": 1989804032 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040385858585858587, + "loss": 1.2137, + "theoretical_loss": 3.43184701456371, + "tokens_seen": 1989869568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004038383838383838, + "loss": 1.449, + "theoretical_loss": 3.431837581386139, + "tokens_seen": 1989935104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004038181818181818, + "loss": 1.2781, + "theoretical_loss": 3.4318281486062174, + "tokens_seen": 1990000640 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040379797979797985, + "loss": 1.7174, + "theoretical_loss": 3.4318187162239147, + "tokens_seen": 1990066176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037777777777778, + "loss": 1.3277, + "theoretical_loss": 3.4318092842392005, + "tokens_seen": 1990131712 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040375757575757576, + "loss": 1.4206, + "theoretical_loss": 3.4317998526520466, + "tokens_seen": 1990197248 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037373737373737, + "loss": 1.425, + "theoretical_loss": 3.4317904214624217, + "tokens_seen": 1990262784 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040371717171717173, + "loss": 1.4692, + "theoretical_loss": 3.4317809906702967, + "tokens_seen": 1990328320 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040369696969696974, + "loss": 1.345, + "theoretical_loss": 3.4317715602756413, + "tokens_seen": 1990393856 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036767676767677, + "loss": 1.3247, + "theoretical_loss": 3.4317621302784262, + "tokens_seen": 1990459392 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040365656565656565, + "loss": 1.5162, + "theoretical_loss": 3.4317527006786217, + "tokens_seen": 1990524928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036363636363636, + "loss": 1.4397, + "theoretical_loss": 3.431743271476197, + "tokens_seen": 1990590464 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036161616161616, + "loss": 1.5959, + "theoretical_loss": 3.4317338426711226, + "tokens_seen": 1990656000 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040359595959595963, + "loss": 1.3966, + "theoretical_loss": 3.4317244142633694, + "tokens_seen": 1990721536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035757575757576, + "loss": 1.3737, + "theoretical_loss": 3.4317149862529073, + "tokens_seen": 1990787072 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040355555555555555, + "loss": 1.4397, + "theoretical_loss": 3.4317055586397056, + "tokens_seen": 1990852608 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.2753543555736542, + "objective/train/avg_token_score": 0.040420789271593094, + "objective/train/docs_used": 1126404, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 0.8407698273658752, + "objective/train/original_loss": 5.04019832611084, + "objective/train/theoretical_loss": 3.4316961314237355, + "objective/train/tokens_used": 349777376, + "objective/train/unlikelihood_loss": 0.060525793582201004, + "theoretical_loss": 3.4316961314237355, + "tokens_seen": 1990918144 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040353535353535356, + "loss": 1.468, + "theoretical_loss": 3.4316961314237355, + "tokens_seen": 1990918144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035151515151515, + "loss": 1.349, + "theoretical_loss": 3.431686704604967, + "tokens_seen": 1990983680 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034949494949495, + "loss": 1.2932, + "theoretical_loss": 3.4316772781833693, + "tokens_seen": 1991049216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034747474747475, + "loss": 1.5111, + "theoretical_loss": 3.431667852158914, + "tokens_seen": 1991114752 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040345454545454544, + "loss": 1.5132, + "theoretical_loss": 3.4316584265315706, + "tokens_seen": 1991180288 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040343434343434345, + "loss": 1.6366, + "theoretical_loss": 3.43164900130131, + "tokens_seen": 1991245824 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034141414141414, + "loss": 1.3234, + "theoretical_loss": 3.4316395764681014, + "tokens_seen": 1991311360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033939393939394, + "loss": 1.3436, + "theoretical_loss": 3.4316301520319152, + "tokens_seen": 1991376896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033737373737374, + "loss": 1.3677, + "theoretical_loss": 3.4316207279927218, + "tokens_seen": 1991442432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033535353535354, + "loss": 1.3794, + "theoretical_loss": 3.4316113043504917, + "tokens_seen": 1991507968 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040333333333333334, + "loss": 1.1624, + "theoretical_loss": 3.4316018811051947, + "tokens_seen": 1991573504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033131313131313, + "loss": 1.4437, + "theoretical_loss": 3.4315924582568016, + "tokens_seen": 1991639040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032929292929293, + "loss": 1.0514, + "theoretical_loss": 3.4315830358052817, + "tokens_seen": 1991704576 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040327272727272727, + "loss": 1.433, + "theoretical_loss": 3.431573613750606, + "tokens_seen": 1991770112 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032525252525253, + "loss": 1.3241, + "theoretical_loss": 3.431564192092744, + "tokens_seen": 1991835648 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040323232323232324, + "loss": 1.4644, + "theoretical_loss": 3.431554770831667, + "tokens_seen": 1991901184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032121212121212, + "loss": 1.4245, + "theoretical_loss": 3.4315453499673443, + "tokens_seen": 1991966720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031919191919192, + "loss": 1.5533, + "theoretical_loss": 3.4315359294997467, + "tokens_seen": 1992032256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031717171717172, + "loss": 1.4905, + "theoretical_loss": 3.431526509428844, + "tokens_seen": 1992097792 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040315151515151517, + "loss": 1.3922, + "theoretical_loss": 3.431517089754607, + "tokens_seen": 1992163328 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040313131313131313, + "loss": 1.4556, + "theoretical_loss": 3.4315076704770053, + "tokens_seen": 1992228864 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031111111111111, + "loss": 1.4792, + "theoretical_loss": 3.4314982515960093, + "tokens_seen": 1992294400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030909090909091, + "loss": 1.7444, + "theoretical_loss": 3.4314888331115894, + "tokens_seen": 1992359936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030707070707071, + "loss": 1.3701, + "theoretical_loss": 3.4314794150237162, + "tokens_seen": 1992425472 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040305050505050506, + "loss": 1.2929, + "theoretical_loss": 3.4314699973323597, + "tokens_seen": 1992491008 + }, + { + "epoch": 0.2, + "objective/train/aligned_token_count": 0.40243157744407654, + "objective/train/avg_token_score": 0.0219747181981802, + "objective/train/docs_used": 1126980, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/likelihood_loss": 1.4165253639221191, + "objective/train/original_loss": 4.742091655731201, + "objective/train/theoretical_loss": 3.4314605800374895, + "objective/train/tokens_used": 351415776, + "objective/train/unlikelihood_loss": 0.05318494141101837, + "theoretical_loss": 3.4314605800374895, + "tokens_seen": 1992556544 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403030303030303, + "loss": 1.4698, + "theoretical_loss": 3.4314605800374895, + "tokens_seen": 1992556544 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403010101010101, + "loss": 1.3088, + "theoretical_loss": 3.4314511631390774, + "tokens_seen": 1992622080 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040298989898989904, + "loss": 1.3889, + "theoretical_loss": 3.431441746637092, + "tokens_seen": 1992687616 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402969696969697, + "loss": 1.4465, + "theoretical_loss": 3.4314323305315044, + "tokens_seen": 1992753152 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040294949494949496, + "loss": 1.3891, + "theoretical_loss": 3.431422914822285, + "tokens_seen": 1992818688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029292929292929, + "loss": 1.5381, + "theoretical_loss": 3.431413499509404, + "tokens_seen": 1992884224 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040290909090909087, + "loss": 1.5889, + "theoretical_loss": 3.431404084592831, + "tokens_seen": 1992949760 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040288888888888894, + "loss": 1.3384, + "theoretical_loss": 3.4313946700725375, + "tokens_seen": 1993015296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028686868686869, + "loss": 1.1691, + "theoretical_loss": 3.4313852559484928, + "tokens_seen": 1993080832 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040284848484848485, + "loss": 1.2463, + "theoretical_loss": 3.4313758422206675, + "tokens_seen": 1993146368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028282828282828, + "loss": 1.4076, + "theoretical_loss": 3.4313664288890324, + "tokens_seen": 1993211904 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040280808080808076, + "loss": 1.5663, + "theoretical_loss": 3.431357015953557, + "tokens_seen": 1993277440 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040278787878787883, + "loss": 1.505, + "theoretical_loss": 3.4313476034142116, + "tokens_seen": 1993342976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027676767676768, + "loss": 1.3878, + "theoretical_loss": 3.431338191270967, + "tokens_seen": 1993408512 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040274747474747474, + "loss": 1.7233, + "theoretical_loss": 3.4313287795237932, + "tokens_seen": 1993474048 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027272727272727, + "loss": 1.4578, + "theoretical_loss": 3.4313193681726606, + "tokens_seen": 1993539584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027070707070707, + "loss": 1.3499, + "theoretical_loss": 3.4313099572175405, + "tokens_seen": 1993605120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026868686868687, + "loss": 1.5321, + "theoretical_loss": 3.4313005466584015, + "tokens_seen": 1993670656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026666666666667, + "loss": 1.6351, + "theoretical_loss": 3.431291136495215, + "tokens_seen": 1993736192 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040264646464646464, + "loss": 1.1717, + "theoretical_loss": 3.4312817267279505, + "tokens_seen": 1993801728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026262626262626, + "loss": 1.3782, + "theoretical_loss": 3.431272317356579, + "tokens_seen": 1993867264 + } + ], + "max_steps": 25000, + "num_train_epochs": 9223372036854775807, + "total_flos": 1.6956807708672e+17, + "trial_name": null, + "trial_params": null +}